Commit 0fc002df authored by huchen's avatar huchen
Browse files

init the dlexamples new

parent 0e04b692
@echo off
:: The conda and wheels jobs are seperated on Windows, so we don't need to clone again.
IF "%BUILD_VISION%" == "" (
if exist "%NIGHTLIES_PYTORCH_ROOT%" (
xcopy /E /Y /Q "%NIGHTLIES_PYTORCH_ROOT%" pytorch\
cd pytorch
goto submodule
)
)
git clone https://github.com/%PYTORCH_REPO%/%MODULE_NAME%
cd %MODULE_NAME%
IF NOT "%BUILD_VISION%" == "" goto latest_end
IF "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end )
:latest_start
if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end )
:date_start
set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'"
set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'"
FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i
FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i
:date_end
if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2%
:: Switch to the latest commit by 11:59 yesterday
echo PYTORCH_BRANCH is set to latest so I will find the last commit
echo before 0:00 midnight on %NIGHTLIES_DATE%
set git_date=%NIGHTLIES_DATE:_=-%
FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i
echo Setting PYTORCH_BRANCH to %last_commit% since that was the last
echo commit before %NIGHTLIES_DATE%
set PYTORCH_BRANCH=%last_commit%
:latest_end
IF "%PYTORCH_BRANCH%" == "" (
set PYTORCH_BRANCH=v%TORCHVISION_BUILD_VERSION%
)
git checkout %PYTORCH_BRANCH%
IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH%
:submodule
git submodule update --init --recursive
IF ERRORLEVEL 1 exit /b 1
copy "%CUDA_PATH%\bin\cusparse64_%CUDA_VERSION%.dll*" pytorch\torch\lib
copy "%CUDA_PATH%\bin\cublas64_%CUDA_VERSION%.dll*" pytorch\torch\lib
copy "%CUDA_PATH%\bin\cudart64_%CUDA_VERSION%.dll*" pytorch\torch\lib
copy "%CUDA_PATH%\bin\curand64_%CUDA_VERSION%.dll*" pytorch\torch\lib
copy "%CUDA_PATH%\bin\cufft64_%CUDA_VERSION%.dll*" pytorch\torch\lib
copy "%CUDA_PATH%\bin\cufftw64_%CUDA_VERSION%.dll*" pytorch\torch\lib
copy "%CUDA_PATH%\bin\cudnn64_%CUDNN_VERSION%.dll*" pytorch\torch\lib
copy "%CUDA_PATH%\bin\nvrtc64_%CUDA_VERSION%*.dll*" pytorch\torch\lib
copy "%CUDA_PATH%\bin\nvrtc-builtins64_%CUDA_VERSION%.dll*" pytorch\torch\lib
copy "C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64\nvToolsExt64_1.dll*" pytorch\torch\lib
copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib
@echo on
if "%CU_VERSION%" == "cpu" (
echo Skipping for CPU builds
exit /b 0
)
set SRC_DIR=%~dp0\..
if not exist "%SRC_DIR%\temp_build" mkdir "%SRC_DIR%\temp_build"
set /a CUDA_VER=%CU_VERSION:cu=%
set CUDA_VER_MAJOR=%CUDA_VER:~0,-1%
set CUDA_VER_MINOR=%CUDA_VER:~-1,1%
set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR%
if %CUDA_VER% EQU 92 goto cuda92
if %CUDA_VER% EQU 100 goto cuda100
if %CUDA_VER% EQU 101 goto cuda101
if %CUDA_VER% EQU 102 goto cuda102
if %CUDA_VER% EQU 110 goto cuda110
echo CUDA %CUDA_VERSION_STR% is not supported
exit /b 1
:cuda92
if not exist "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" (
curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_9.2.148_win10.exe --output "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe"
if errorlevel 1 exit /b 1
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe"
set "ARGS=nvcc_9.2 cuobjdump_9.2 nvprune_9.2 cupti_9.2 cublas_9.2 cublas_dev_9.2 cudart_9.2 cufft_9.2 cufft_dev_9.2 curand_9.2 curand_dev_9.2 cusolver_9.2 cusolver_dev_9.2 cusparse_9.2 cusparse_dev_9.2 nvgraph_9.2 nvgraph_dev_9.2 npp_9.2 npp_dev_9.2 nvrtc_9.2 nvrtc_dev_9.2 nvml_dev_9.2"
)
if not exist "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" (
curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-9.2-windows10-x64-v7.2.1.38.zip --output "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip"
if errorlevel 1 exit /b 1
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip"
)
goto cuda_common
:cuda100
if not exist "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" (
curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_10.0.130_411.31_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe"
if errorlevel 1 exit /b 1
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe"
set "ARGS=nvcc_10.0 cuobjdump_10.0 nvprune_10.0 cupti_10.0 cublas_10.0 cublas_dev_10.0 cudart_10.0 cufft_10.0 cufft_dev_10.0 curand_10.0 curand_dev_10.0 cusolver_10.0 cusolver_dev_10.0 cusparse_10.0 cusparse_dev_10.0 nvgraph_10.0 nvgraph_dev_10.0 npp_10.0 npp_dev_10.0 nvrtc_10.0 nvrtc_dev_10.0 nvml_dev_10.0"
)
if not exist "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" (
curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-10.0-windows10-x64-v7.4.1.5.zip --output "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip"
if errorlevel 1 exit /b 1
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip"
)
goto cuda_common
:cuda101
if not exist "%SRC_DIR%\temp_build\cuda_10.1.243_426.00_win10.exe" (
curl -k -L https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.1.243_426.00_win10.exe"
if errorlevel 1 exit /b 1
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.1.243_426.00_win10.exe"
set "ARGS=nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1"
)
if not exist "%SRC_DIR%\temp_build\cudnn-10.1-windows10-x64-v7.6.4.38.zip" (
curl -k -L https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip --output "%SRC_DIR%\temp_build\cudnn-10.1-windows10-x64-v7.6.4.38.zip"
if errorlevel 1 exit /b 1
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.1-windows10-x64-v7.6.4.38.zip"
)
goto cuda_common
:cuda102
if not exist "%SRC_DIR%\temp_build\cuda_10.2.89_441.22_win10.exe" (
curl -k -L https://ossci-windows.s3.amazonaws.com/cuda_10.2.89_441.22_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.2.89_441.22_win10.exe"
if errorlevel 1 exit /b 1
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.2.89_441.22_win10.exe"
set "ARGS=nvcc_10.2 cuobjdump_10.2 nvprune_10.2 cupti_10.2 cublas_10.2 cublas_dev_10.2 cudart_10.2 cufft_10.2 cufft_dev_10.2 curand_10.2 curand_dev_10.2 cusolver_10.2 cusolver_dev_10.2 cusparse_10.2 cusparse_dev_10.2 nvgraph_10.2 nvgraph_dev_10.2 npp_10.2 npp_dev_10.2 nvrtc_10.2 nvrtc_dev_10.2 nvml_dev_10.2"
)
if not exist "%SRC_DIR%\temp_build\cudnn-10.2-windows10-x64-v7.6.5.32.zip" (
curl -k -L https://ossci-windows.s3.amazonaws.com/cudnn-10.2-windows10-x64-v7.6.5.32.zip --output "%SRC_DIR%\temp_build\cudnn-10.2-windows10-x64-v7.6.5.32.zip"
if errorlevel 1 exit /b 1
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.2-windows10-x64-v7.6.5.32.zip"
)
goto cuda_common
:cuda110
if not exist "%SRC_DIR%\temp_build\cuda_11.0.2_451.48_win10.exe" (
curl -k -L https://ossci-windows.s3.amazonaws.com/cuda_11.0.2_451.48_win10.exe --output "%SRC_DIR%\temp_build\cuda_11.0.2_451.48_win10.exe"
if errorlevel 1 exit /b 1
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_11.0.2_451.48_win10.exe"
set "ARGS=nvcc_11.0 cuobjdump_11.0 nvprune_11.0 nvprof_11.0 cupti_11.0 cublas_11.0 cublas_dev_11.0 cudart_11.0 cufft_11.0 cufft_dev_11.0 curand_11.0 curand_dev_11.0 cusolver_11.0 cusolver_dev_11.0 cusparse_11.0 cusparse_dev_11.0 npp_11.0 npp_dev_11.0 nvrtc_11.0 nvrtc_dev_11.0 nvml_dev_11.0"
)
if not exist "%SRC_DIR%\temp_build\cudnn-11.0-windows-x64-v8.0.4.30.zip" (
curl -k -L https://ossci-windows.s3.amazonaws.com/cudnn-11.0-windows-x64-v8.0.4.30.zip --output "%SRC_DIR%\temp_build\cudnn-11.0-windows-x64-v8.0.4.30.zip"
if errorlevel 1 exit /b 1
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-11.0-windows-x64-v8.0.4.30.zip"
)
goto cuda_common
:cuda_common
if not exist "%SRC_DIR%\temp_build\NvToolsExt.7z" (
curl -k -L https://www.dropbox.com/s/9mcolalfdj4n979/NvToolsExt.7z?dl=1 --output "%SRC_DIR%\temp_build\NvToolsExt.7z"
if errorlevel 1 exit /b 1
)
if not exist "%SRC_DIR%\temp_build\gpu_driver_dlls.7z" (
curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "%SRC_DIR%\temp_build\gpu_driver_dlls.zip"
if errorlevel 1 exit /b 1
)
echo Installing CUDA toolkit...
7z x %CUDA_SETUP_FILE% -o"%SRC_DIR%\temp_build\cuda"
pushd "%SRC_DIR%\temp_build\cuda"
start /wait setup.exe -s %ARGS%
popd
echo Installing VS integration...
xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\IDE\VC\VCTargets\BuildCustomizations"
echo Installing NvToolsExt...
7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt"
mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64"
mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include"
mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64"
xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\bin\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64"
xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\include\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include"
xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\lib\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64"
echo Setting up environment...
set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%"
set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64"
if not exist "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin\nvcc.exe" (
echo CUDA %CUDA_VERSION_STR% installed failed.
exit /b 1
)
echo Installing cuDNN...
7z x %CUDNN_SETUP_FILE% -o"%SRC_DIR%\temp_build\cudnn"
xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin"
xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\lib\x64\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64"
xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include"
echo Installing GPU driver DLLs
7z x %SRC_DIR%\temp_build\gpu_driver_dlls.zip -o"C:\Windows\System32"
echo Cleaning temp files
rd /s /q "%SRC_DIR%\temp_build" || ver > nul
@echo off
REM curl -k https://www.7-zip.org/a/7z1805-x64.exe -O
REM if errorlevel 1 exit /b 1
REM start /wait 7z1805-x64.exe /S
REM if errorlevel 1 exit /b 1
REM set "PATH=%ProgramFiles%\7-Zip;%PATH%"
choco feature disable --name showDownloadProgress
choco feature enable --name allowGlobalConfirmation
choco install curl 7zip
@echo off
:: Caution: Please don't use this script locally
:: It may destroy your build environment.
setlocal
IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" (
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
exit /b 1
)
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VS15INSTALLDIR=%%i"
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
goto vswhere
)
)
:vswhere
IF "%VS15VCVARSALL%"=="" (
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
exit /b 1
)
call "%VS15VCVARSALL%" x86_amd64
for /f "usebackq tokens=*" %%i in (`where link.exe`) do move "%%i" "%%i.bak"
endlocal
@echo on
if "%~1"=="" goto arg_error
if NOT "%~2"=="" goto arg_error
goto arg_end
:arg_error
echo Illegal number of parameters. Pass packge type `Conda` or `Wheels`.
exit /b 1
:arg_end
echo "nightly_defaults.bat at %CD% starting at %DATE%"
set SRC_DIR=%~dp0\..
:: NIGHTLIES_FOLDER
:: N.B. this is also defined in cron_start.sh
:: An arbitrary root folder to store all nightlies folders, each of which is a
:: parent level date folder with separate subdirs for logs, wheels, conda
:: packages, etc. This should be kept the same across all scripts called in a
:: cron job, so it only has a default value in the top-most script
:: build_cron.sh to avoid the default values from diverging.
if "%NIGHTLIES_FOLDER%" == "" set "NIGHTLIES_FOLDER=%SRC_DIR%"
:: NIGHTLIES_DATE
:: N.B. this is also defined in cron_start.sh
:: The date in YYYY_mm_dd format that we are building for. If this is not
:: already set, then this will first try to find the date of the nightlies
:: folder that this builder repo exists in; e.g. if this script exists in
:: some_dir/2019_09_04/builder/cron/ then this will be set to 2019_09_04 (must
:: match YYYY_mm_dd). This is for convenience when debugging/uploading past
:: dates, so that you don't have to set NIGHTLIES_DATE yourself. If a date
:: folder cannot be found in that exact location, then this will default to
:: the current date.
if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end )
:date_start
set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'"
set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'"
FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i
FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i
:date_end
if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2%
:: Used in lots of places as the root dir to store all conda/wheel/manywheel
:: packages as well as logs for the day
set today=%NIGHTLIES_FOLDER%\%NIGHTLIES_DATE%
mkdir "%today%" || ver >nul
::#############################################################################
:: Add new configuration variables below this line. 'today' should always be
:: defined ASAP to avoid weird errors
::#############################################################################
:: List of people to email when things go wrong. This is passed directly to
:: `mail -t`
:: TODO: Not supported yet
if "%NIGHTLIES_EMAIL_LIST%" == "" set NIGHTLIES_EMAIL_LIST=peterghost86@gmail.com
:: PYTORCH_CREDENTIALS_FILE
:: A bash file that exports credentials needed to upload to aws and anaconda.
:: Needed variables are PYTORCH_ANACONDA_USERNAME, PYTORCH_ANACONDA_PASSWORD,
:: AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Or it can just export the AWS
:: keys and then prepend a logged-in conda installation to the path.
:: TODO: Not supported yet
if "%PYTORCH_CREDENTIALS_FILE%" == "" set PYTORCH_CREDENTIALS_FILE=/c/Users/administrator/nightlies/credentials.sh
:: Location of the temporary miniconda that is downloaded to install conda-build
:: and aws to upload finished packages TODO this is messy to install this in
:: upload.sh and later use it in upload_logs.sh
if "%CONDA_UPLOADER_INSTALLATION%" == "" set "CONDA_UPLOADER_INSTALLATION=%today%\miniconda"
:: N.B. BUILDER_REPO and BUILDER_BRANCH are both set in cron_start.sh, as that
:: is the script that actually clones the builder repo that /this/ script is
:: running from.
pushd "%SRC_DIR%\.."
set NIGHTLIES_BUILDER_ROOT=%CD%
popd
:: The shared pytorch repo to be used by all builds
if "%NIGHTLIES_PYTORCH_ROOT%" == "" set "NIGHTLIES_PYTORCH_ROOT=%today%\vision"
:: PYTORCH_REPO
:: The Github org/user whose fork of Pytorch to check out (git clone
:: https://github.com/<THIS_PART>/pytorch.git). This will always be cloned
:: fresh to build with. Default is 'pytorch'
if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch
:: PYTORCH_BRANCH
:: The branch of Pytorch to checkout for building (git checkout <THIS_PART>).
:: This can either be the name of the branch (e.g. git checkout
:: my_branch_name) or can be a git commit (git checkout 4b2674n...). Default
:: is 'latest', which is a special term that signals to pull the last commit
:: before 0:00 midnight on the NIGHTLIES_DATE
if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=nightly
:: Clone the requested pytorch checkout
if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start )
:clone_start
git clone --recursive "https://github.com/%PYTORCH_REPO%/vision.git" "%NIGHTLIES_PYTORCH_ROOT%"
pushd "%NIGHTLIES_PYTORCH_ROOT%"
if "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end )
:latest_start
:: Switch to the latest commit by 11:59 yesterday
echo PYTORCH_BRANCH is set to latest so I will find the last commit
echo before 0:00 midnight on %NIGHTLIES_DATE%
set git_date=%NIGHTLIES_DATE:_=-%
FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i
echo Setting PYTORCH_BRANCH to %last_commit% since that was the last
echo commit before %NIGHTLIES_DATE%
set PYTORCH_BRANCH=%last_commit%
:latest_end
git checkout "%PYTORCH_BRANCH%"
git submodule update
popd
:clone_end
if "%CUDA_VERSION%" == "cpu" (
set _DESIRED_CUDA=cpu
) else (
set _DESIRED_CUDA=cu%CUDA_VERSION%
)
:: PYTORCH_BUILD_VERSION
:: The actual version string. Used in conda like
:: pytorch-nightly==1.0.0.dev20180908
:: or in manylinux like
:: torch_nightly-1.0.0.dev20180908-cp27-cp27m-linux_x86_64.whl
if "%TORCHVISION_BUILD_VERSION%" == "" set TORCHVISION_BUILD_VERSION=0.9.0.dev%NIGHTLIES_DATE_COMPACT%
if "%~1" == "Wheels" (
if not "%CUDA_VERSION%" == "102" (
set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%+%_DESIRED_CUDA%
)
)
:: PYTORCH_BUILD_NUMBER
:: This is usually the number 1. If more than one build is uploaded for the
:: same version/date, then this can be incremented to 2,3 etc in which case
:: '.post2' will be appended to the version string of the package. This can
:: be set to '0' only if OVERRIDE_PACKAGE_VERSION is being used to bypass
:: all the version string logic in downstream scripts. Since we use the
:: override below, exporting this shouldn't actually matter.
if "%TORCHVISION_BUILD_NUMBER%" == "" set /a TORCHVISION_BUILD_NUMBER=1
if %TORCHVISION_BUILD_NUMBER% GTR 1 set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%%TORCHVISION_BUILD_NUMBER%
:: The nightly builds use their own versioning logic, so we override whatever
:: logic is in setup.py or other scripts
:: TODO: Not supported yet
set OVERRIDE_PACKAGE_VERSION=%TORCHVISION_BUILD_VERSION%
set BUILD_VERSION=%TORCHVISION_BUILD_VERSION%
:: Build folder for conda builds to use
if "%TORCH_CONDA_BUILD_FOLDER%" == "" set TORCH_CONDA_BUILD_FOLDER=torchvision
:: TORCH_PACKAGE_NAME
:: The name of the package to upload. This should probably be pytorch or
:: pytorch-nightly. N.B. that pip will change all '-' to '_' but conda will
:: not. This is dealt with in downstream scripts.
:: TODO: Not supported yet
if "%TORCH_PACKAGE_NAME%" == "" set TORCH_PACKAGE_NAME=torchvision
:: PIP_UPLOAD_FOLDER should end in a slash. This is to handle it being empty
:: (when uploading to e.g. whl/cpu/) and also to handle nightlies (when
:: uploading to e.g. /whl/nightly/cpu)
:: TODO: Not supported yet
if "%PIP_UPLOAD_FOLDER%" == "" set "PIP_UPLOAD_FOLDER=nightly\"
:: The location of the binary_sizes dir in s3 is hardcoded into
:: upload_binary_sizes.sh
:: DAYS_TO_KEEP
:: How many days to keep around for clean.sh. Build folders older than this
:: will be purged at the end of cron jobs. '1' means to keep only the current
:: day. Values less than 1 are not allowed. The default is 5.
:: TODO: Not supported yet
if "%DAYS_TO_KEEP%" == "" set /a DAYS_TO_KEEP=5
if %DAYS_TO_KEEP% LSS 1 (
echo DAYS_TO_KEEP cannot be less than 1.
echo A value of 1 means to only keep the build for today
exit /b 1
)
@echo off
set SRC_DIR=%~dp0
pushd %SRC_DIR%
if NOT "%CUDA_VERSION%" == "cpu" (
set PACKAGE_SUFFIX=_cuda%CUDA_VERSION%
) else (
set PACKAGE_SUFFIX=
)
if "%PACKAGEFULLNAME%" == "Conda" (
set PACKAGE=conda
) else (
set PACKAGE=wheels
)
if not defined PACKAGE_SUFFIX (
set PUBLISH_BRANCH=vision_%PACKAGE%_%DESIRED_PYTHON%
) else (
set PUBLISH_BRANCH=vision_%PACKAGE%_%DESIRED_PYTHON%%PACKAGE_SUFFIX%
)
git clone %ARTIFACT_REPO_URL% -b %PUBLISH_BRANCH% --single-branch >nul 2>&1
IF ERRORLEVEL 1 (
echo Branch %PUBLISH_BRANCH% not exist, falling back to master
set NO_BRANCH=1
git clone %ARTIFACT_REPO_URL% -b master --single-branch >nul 2>&1
)
IF ERRORLEVEL 1 (
echo Clone failed
goto err
)
cd pytorch_builder
attrib -s -h -r . /s /d
:: Empty repo
rd /s /q . || ver >nul
IF NOT EXIST %PACKAGE% mkdir %PACKAGE%
xcopy /S /E /Y ..\..\output\*.* %PACKAGE%\
git config --global user.name "Azure DevOps"
git config --global user.email peterghost86@gmail.com
git init
git checkout --orphan %PUBLISH_BRANCH%
git remote add origin %ARTIFACT_REPO_URL%
git add .
git commit -m "Update artifacts"
:push
if "%RETRY_TIMES%" == "" (
set /a RETRY_TIMES=10
set /a SLEEP_TIME=2
) else (
set /a RETRY_TIMES=%RETRY_TIMES%-1
set /a SLEEP_TIME=%SLEEP_TIME%*2
)
git push origin %PUBLISH_BRANCH% -f > nul 2>&1
IF ERRORLEVEL 1 (
echo Git push retry times remaining: %RETRY_TIMES%
echo Sleep time: %SLEEP_TIME% seconds
IF %RETRY_TIMES% EQU 0 (
echo Push failed
goto err
)
waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
goto push
) ELSE (
set RETRY_TIMES=
set SLEEP_TIME=
)
popd
exit /b 0
:err
popd
exit /b 1
@echo off
echo The flags after configuring:
echo NO_CUDA=%NO_CUDA%
echo CMAKE_GENERATOR=%CMAKE_GENERATOR%
if "%NO_CUDA%"=="" echo CUDA_PATH=%CUDA_PATH%
if NOT "%CC%"=="" echo CC=%CC%
if NOT "%CXX%"=="" echo CXX=%CXX%
if NOT "%DISTUTILS_USE_SDK%"=="" echo DISTUTILS_USE_SDK=%DISTUTILS_USE_SDK%
set SRC_DIR=%~dp0\..
IF "%VSDEVCMD_ARGS%" == "" (
call "%VS15VCVARSALL%" x64
) ELSE (
call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS%
)
pushd %SRC_DIR%
IF NOT exist "setup.py" (
cd %MODULE_NAME%
)
if "%CXX%"=="sccache cl" (
sccache --stop-server
sccache --start-server
sccache --zero-stats
)
:pytorch
:: This stores in e.g. D:/_work/1/s/windows/output/cpu
pip wheel -e . --no-deps --wheel-dir ../output
:build_end
IF ERRORLEVEL 1 exit /b 1
IF NOT ERRORLEVEL 0 exit /b 1
if "%CXX%"=="sccache cl" (
taskkill /im sccache.exe /f /t || ver > nul
taskkill /im nvcc.exe /f /t || ver > nul
)
cd ..
@echo off
set SRC_DIR=%~dp0\..
pushd %SRC_DIR%
set PYTHON_VERSION=%PYTHON_PREFIX:py=cp%
if "%BUILD_VISION%" == "" (
pip install future pytest coverage hypothesis protobuf
) ELSE (
pip install future pytest "pillow>=4.1.1"
)
for /F "delims=" %%i in ('where /R %SRC_DIR%\output *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i"
if ERRORLEVEL 1 exit /b 1
if NOT "%BUILD_VISION%" == "" (
echo Smoke testing imports
python -c "import torchvision"
if ERRORLEVEL 1 exit /b 1
goto smoke_test_end
)
echo Smoke testing imports
python -c "import torch"
if ERRORLEVEL 1 exit /b 1
python -c "from caffe2.python import core"
if ERRORLEVEL 1 exit /b 1
echo Checking that MKL is available
python -c "import torch; exit(0 if torch.backends.mkl.is_available() else 1)"
if ERRORLEVEL 1 exit /b 1
setlocal EnableDelayedExpansion
set NVIDIA_GPU_EXISTS=0
for /F "delims=" %%i in ('wmic path win32_VideoController get name') do (
set GPUS=%%i
if not "x!GPUS:NVIDIA=!" == "x!GPUS!" (
SET NVIDIA_GPU_EXISTS=1
goto gpu_check_end
)
)
:gpu_check_end
endlocal & set NVIDIA_GPU_EXISTS=%NVIDIA_GPU_EXISTS%
if NOT "%CUDA_PREFIX%" == "cpu" if "%NVIDIA_GPU_EXISTS%" == "1" (
echo Checking that CUDA archs are setup correctly
python -c "import torch; torch.randn([3,5]).cuda()"
if ERRORLEVEL 1 exit /b 1
echo Checking that magma is available
python -c "import torch; torch.rand(1).cuda(); exit(0 if torch.cuda.has_magma else 1)"
if ERRORLEVEL 1 exit /b 1
echo Checking that CuDNN is available
python -c "import torch; exit(0 if torch.backends.cudnn.is_available() else 1)"
if ERRORLEVEL 1 exit /b 1
)
:smoke_test_end
echo Not running unit tests. Hopefully these problems are caught by CI
goto test_end
if "%BUILD_VISION%" == "" (
cd pytorch\test
python run_test.py -v
) else (
cd vision
pytest .
)
if ERRORLEVEL 1 exit /b 1
:test_end
popd
exit /b 0
@echo off
IF "%CONDA_UPLOADER_INSTALLATION%" == "" goto precheck_fail
IF "%PYTORCH_FINAL_PACKAGE_DIR%" == "" goto precheck_fail
IF "%today%" == "" goto precheck_fail
IF "%PYTORCH_ANACONDA_USERNAME%" == "" goto precheck_fail
IF "%PYTORCH_ANACONDA_PASSWORD%" == "" goto precheck_fail
goto precheck_pass
:precheck_fail
echo Please run nightly_defaults.bat first.
echo And remember to set `PYTORCH_FINAL_PACKAGE_DIR`
echo Finally, don't forget to set anaconda tokens
exit /b 1
:precheck_pass
pushd %today%
:: Install anaconda client
set "CONDA_HOME=%CONDA_UPLOADER_INSTALLATION%"
set "tmp_conda=%CONDA_HOME%"
set "miniconda_exe=%CD%\miniconda.exe"
rmdir /s /q "%CONDA_HOME%"
del miniconda.exe
curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%"
popd
IF ERRORLEVEL 1 (
echo Conda download failed
exit /b 1
)
call %~dp0\..\..\conda\install_conda.bat
IF ERRORLEVEL 1 (
echo Conda installation failed
exit /b 1
)
set "ORIG_PATH=%PATH%"
set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
REM conda install -y anaconda-client
pip install git+https://github.com/peterjc123/anaconda-client.git@log_more_meaningfull_errors
IF ERRORLEVEL 1 (
echo Anaconda client installation failed
exit /b 1
)
set PYTORCH_FINAL_PACKAGE=
:: Upload all the packages under `PYTORCH_FINAL_PACKAGE_DIR`
FOR /F "delims=" %%i IN ('where /R %PYTORCH_FINAL_PACKAGE_DIR% *vision*.tar.bz2') DO (
set "PYTORCH_FINAL_PACKAGE=%%i"
)
IF "%PYTORCH_FINAL_PACKAGE%" == "" (
echo No package to upload
exit /b 0
)
:upload
if "%RETRY_TIMES%" == "" (
set /a RETRY_TIMES=10
set /a SLEEP_TIME=2
) else (
set /a RETRY_TIMES=%RETRY_TIMES%-1
set /a SLEEP_TIME=%SLEEP_TIME%*2
)
REM bash -c "yes | anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%""
anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%"
IF ERRORLEVEL 1 (
echo Anaconda client login failed
exit /b 1
)
echo Uploading %PYTORCH_FINAL_PACKAGE% to Anaconda Cloud
anaconda upload "%PYTORCH_FINAL_PACKAGE%" -u pytorch-nightly --label main --force --no-progress
IF ERRORLEVEL 1 (
echo Anaconda upload retry times remaining: %RETRY_TIMES%
echo Sleep time: %SLEEP_TIME% seconds
IF %RETRY_TIMES% EQU 0 (
echo Upload failed
exit /b 1
)
waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
goto upload
) ELSE (
set RETRY_TIMES=
set SLEEP_TIME=
)
@echo on
set VC_VERSION_LOWER=16
set VC_VERSION_UPPER=17
if "%VC_YEAR%" == "2017" (
set VC_VERSION_LOWER=15
set VC_VERSION_UPPER=16
)
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VS15INSTALLDIR=%%i"
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
goto vswhere
)
)
:vswhere
if "%VSDEVCMD_ARGS%" == "" (
call "%VS15VCVARSALL%" x64 || exit /b 1
) else (
call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1
)
@echo on
set DISTUTILS_USE_SDK=1
set args=%1
shift
:start
if [%1] == [] goto done
set args=%args% %1
shift
goto start
:done
if "%args%" == "" (
echo Usage: vc_env_helper.bat [command] [args]
echo e.g. vc_env_helper.bat cl /c test.cpp
)
%args% || exit /b 1
#!/bin/bash
set -ex
if [[ "$CU_VERSION" == "cu92" ]]; then
export VC_YEAR=2017
export VSDEVCMD_ARGS="-vcvars_ver=14.13"
powershell packaging/windows/internal/vs2017_install.ps1
elif [[ "$CU_VERSION" == "cu100" ]]; then
export VC_YEAR=2017
export VSDEVCMD_ARGS=""
powershell packaging/windows/internal/vs2017_install.ps1
else
export VC_YEAR=2019
export VSDEVCMD_ARGS=""
fi
$VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
"--add Microsoft.VisualStudio.Component.VC.Tools.14.13",
"--add Microsoft.Component.MSBuild",
"--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
"--add Microsoft.VisualStudio.Component.TextTemplating",
"--add Microsoft.VisualStudio.Component.VC.CoreIde",
"--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
"--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81")
curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
if ($LASTEXITCODE -ne 0) {
echo "Download of the VS 2017 installer failed"
exit 1
}
$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
Remove-Item -Path vs_installer.exe -Force
$exitCode = $process.ExitCode
if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
exit 1
}
$VS_DOWNLOAD_LINK = "https://aka.ms/vs/16/release/vs_buildtools.exe"
$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
"--add Microsoft.Component.MSBuild",
"--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
"--add Microsoft.VisualStudio.Component.VC.CoreBuildTools",
"--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
"--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64")
curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
if ($LASTEXITCODE -ne 0) {
echo "Download of the VS 2019 installer failed"
exit 1
}
$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
Remove-Item -Path vs_installer.exe -Force
$exitCode = $process.ExitCode
if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
echo "VS 2019 installer exited with code $exitCode, which should be one of [0, 3010]."
exit 1
}
@echo off
set VS_DOWNLOAD_LINK=https://aka.ms/vs/15/release/vs_enterprise.exe
set VS_INSTALL_PATH=C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise
set VS_INSTALL_ARGS=--nocache --quiet --wait --add Microsoft.VisualStudio.Component.VC.Tools.14.11
set VSDEVCMD_ARGS=-vcvars_ver=14.11
curl -k -L %VS_DOWNLOAD_LINK% --output vs_installer.exe
if errorlevel 1 exit /b 1
start /wait vs_installer.exe modify --installPath "%VS_INSTALL_PATH%" %VS_INSTALL_ARGS%
if not errorlevel 0 exit /b 1
if errorlevel 1 if not errorlevel 3010 exit /b 1
if errorlevel 3011 exit /b 1
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
set NO_CUDA=
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
IF "%NVTOOLSEXT_PATH%"=="" (
echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
exit /b 1
goto optcheck
)
IF "%CUDA_PATH_V10_0%"=="" (
echo CUDA 10.0 not found, failing
exit /b 1
) ELSE (
IF "%BUILD_VISION%" == "" (
set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5
set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
) ELSE (
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50
)
set "CUDA_PATH=%CUDA_PATH_V10_0%"
set "PATH=%CUDA_PATH_V10_0%\bin;%PATH%"
)
:optcheck
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
set NO_CUDA=
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
IF "%NVTOOLSEXT_PATH%"=="" (
echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
exit /b 1
goto optcheck
)
IF "%CUDA_PATH_V9_0%"=="" (
echo CUDA 9 not found, failing
exit /b 1
) ELSE (
IF "%BUILD_VISION%" == "" (
set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;7.0
set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
) ELSE (
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
)
set "CUDA_PATH=%CUDA_PATH_V9_0%"
set "PATH=%CUDA_PATH_V9_0%\bin;%PATH%"
)
:optcheck
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
jobs:
- job: 'VSTS_Auth_Task'
timeoutInMinutes: 5
cancelTimeoutInMinutes: 5
variables:
- group: 'peterjc-vsts-token'
pool:
vmImage: 'vs2017-win2016'
steps:
- checkout: self
clean: true
- template: vsts_auth.yml
parameters:
auth: $(vsts_auth)
parameters:
msagent: false
steps:
- bash: 'find . -name "*.sh" -exec dos2unix {} +'
displayName: Replace file endings
- script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%'
displayName: 'Create final package directory'
- bash: './packaging/conda/build_vision.sh $CUDA_VERSION $TORCHVISION_BUILD_VERSION $TORCHVISION_BUILD_NUMBER'
displayName: Build
env:
${{ if eq(parameters.msagent, 'true') }}:
MAX_JOBS: 2
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment