Unverified Commit 968cefc2 authored by Nicolas Hug's avatar Nicolas Hug Committed by GitHub
Browse files

Remove unneeded packaging files (#3618)



* remove some files

* Also remove entire packaging/conda folder
Co-authored-by: default avatarFrancisco Massa <fvsmassa@gmail.com>
parent f05bfc00
@echo off
:: Caution: Please don't use this script locally
:: It may destroy your build environment.
setlocal
IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" (
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
exit /b 1
)
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VS15INSTALLDIR=%%i"
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
goto vswhere
)
)
:vswhere
IF "%VS15VCVARSALL%"=="" (
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
exit /b 1
)
call "%VS15VCVARSALL%" x86_amd64
for /f "usebackq tokens=*" %%i in (`where link.exe`) do move "%%i" "%%i.bak"
endlocal
@echo on
if "%~1"=="" goto arg_error
if NOT "%~2"=="" goto arg_error
goto arg_end
:arg_error
echo Illegal number of parameters. Pass packge type `Conda` or `Wheels`.
exit /b 1
:arg_end
echo "nightly_defaults.bat at %CD% starting at %DATE%"
set SRC_DIR=%~dp0\..
:: NIGHTLIES_FOLDER
:: N.B. this is also defined in cron_start.sh
:: An arbitrary root folder to store all nightlies folders, each of which is a
:: parent level date folder with separate subdirs for logs, wheels, conda
:: packages, etc. This should be kept the same across all scripts called in a
:: cron job, so it only has a default value in the top-most script
:: build_cron.sh to avoid the default values from diverging.
if "%NIGHTLIES_FOLDER%" == "" set "NIGHTLIES_FOLDER=%SRC_DIR%"
:: NIGHTLIES_DATE
:: N.B. this is also defined in cron_start.sh
:: The date in YYYY_mm_dd format that we are building for. If this is not
:: already set, then this will first try to find the date of the nightlies
:: folder that this builder repo exists in; e.g. if this script exists in
:: some_dir/2019_09_04/builder/cron/ then this will be set to 2019_09_04 (must
:: match YYYY_mm_dd). This is for convenience when debugging/uploading past
:: dates, so that you don't have to set NIGHTLIES_DATE yourself. If a date
:: folder cannot be found in that exact location, then this will default to
:: the current date.
if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end )
:date_start
set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'"
set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'"
FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i
FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i
:date_end
if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2%
:: Used in lots of places as the root dir to store all conda/wheel/manywheel
:: packages as well as logs for the day
set today=%NIGHTLIES_FOLDER%\%NIGHTLIES_DATE%
mkdir "%today%" || ver >nul
::#############################################################################
:: Add new configuration variables below this line. 'today' should always be
:: defined ASAP to avoid weird errors
::#############################################################################
:: List of people to email when things go wrong. This is passed directly to
:: `mail -t`
:: TODO: Not supported yet
if "%NIGHTLIES_EMAIL_LIST%" == "" set NIGHTLIES_EMAIL_LIST=peterghost86@gmail.com
:: PYTORCH_CREDENTIALS_FILE
:: A bash file that exports credentials needed to upload to aws and anaconda.
:: Needed variables are PYTORCH_ANACONDA_USERNAME, PYTORCH_ANACONDA_PASSWORD,
:: AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Or it can just export the AWS
:: keys and then prepend a logged-in conda installation to the path.
:: TODO: Not supported yet
if "%PYTORCH_CREDENTIALS_FILE%" == "" set PYTORCH_CREDENTIALS_FILE=/c/Users/administrator/nightlies/credentials.sh
:: Location of the temporary miniconda that is downloaded to install conda-build
:: and aws to upload finished packages TODO this is messy to install this in
:: upload.sh and later use it in upload_logs.sh
if "%CONDA_UPLOADER_INSTALLATION%" == "" set "CONDA_UPLOADER_INSTALLATION=%today%\miniconda"
:: N.B. BUILDER_REPO and BUILDER_BRANCH are both set in cron_start.sh, as that
:: is the script that actually clones the builder repo that /this/ script is
:: running from.
pushd "%SRC_DIR%\.."
set NIGHTLIES_BUILDER_ROOT=%CD%
popd
:: The shared pytorch repo to be used by all builds
if "%NIGHTLIES_PYTORCH_ROOT%" == "" set "NIGHTLIES_PYTORCH_ROOT=%today%\vision"
:: PYTORCH_REPO
:: The Github org/user whose fork of Pytorch to check out (git clone
:: https://github.com/<THIS_PART>/pytorch.git). This will always be cloned
:: fresh to build with. Default is 'pytorch'
if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch
:: PYTORCH_BRANCH
:: The branch of Pytorch to checkout for building (git checkout <THIS_PART>).
:: This can either be the name of the branch (e.g. git checkout
:: my_branch_name) or can be a git commit (git checkout 4b2674n...). Default
:: is 'latest', which is a special term that signals to pull the last commit
:: before 0:00 midnight on the NIGHTLIES_DATE
if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=nightly
:: Clone the requested pytorch checkout
if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start )
:clone_start
git clone --recursive "https://github.com/%PYTORCH_REPO%/vision.git" "%NIGHTLIES_PYTORCH_ROOT%"
pushd "%NIGHTLIES_PYTORCH_ROOT%"
if "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end )
:latest_start
:: Switch to the latest commit by 11:59 yesterday
echo PYTORCH_BRANCH is set to latest so I will find the last commit
echo before 0:00 midnight on %NIGHTLIES_DATE%
set git_date=%NIGHTLIES_DATE:_=-%
FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i
echo Setting PYTORCH_BRANCH to %last_commit% since that was the last
echo commit before %NIGHTLIES_DATE%
set PYTORCH_BRANCH=%last_commit%
:latest_end
git checkout "%PYTORCH_BRANCH%"
git submodule update
popd
:clone_end
if "%CUDA_VERSION%" == "cpu" (
set _DESIRED_CUDA=cpu
) else (
set _DESIRED_CUDA=cu%CUDA_VERSION%
)
:: PYTORCH_BUILD_VERSION
:: The actual version string. Used in conda like
:: pytorch-nightly==1.0.0.dev20180908
:: or in manylinux like
:: torch_nightly-1.0.0.dev20180908-cp27-cp27m-linux_x86_64.whl
if "%TORCHVISION_BUILD_VERSION%" == "" set TORCHVISION_BUILD_VERSION=0.10.0.dev%NIGHTLIES_DATE_COMPACT%
if "%~1" == "Wheels" (
if not "%CUDA_VERSION%" == "102" (
set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%+%_DESIRED_CUDA%
)
)
:: PYTORCH_BUILD_NUMBER
:: This is usually the number 1. If more than one build is uploaded for the
:: same version/date, then this can be incremented to 2,3 etc in which case
:: '.post2' will be appended to the version string of the package. This can
:: be set to '0' only if OVERRIDE_PACKAGE_VERSION is being used to bypass
:: all the version string logic in downstream scripts. Since we use the
:: override below, exporting this shouldn't actually matter.
if "%TORCHVISION_BUILD_NUMBER%" == "" set /a TORCHVISION_BUILD_NUMBER=1
if %TORCHVISION_BUILD_NUMBER% GTR 1 set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%%TORCHVISION_BUILD_NUMBER%
:: The nightly builds use their own versioning logic, so we override whatever
:: logic is in setup.py or other scripts
:: TODO: Not supported yet
set OVERRIDE_PACKAGE_VERSION=%TORCHVISION_BUILD_VERSION%
set BUILD_VERSION=%TORCHVISION_BUILD_VERSION%
:: Build folder for conda builds to use
if "%TORCH_CONDA_BUILD_FOLDER%" == "" set TORCH_CONDA_BUILD_FOLDER=torchvision
:: TORCH_PACKAGE_NAME
:: The name of the package to upload. This should probably be pytorch or
:: pytorch-nightly. N.B. that pip will change all '-' to '_' but conda will
:: not. This is dealt with in downstream scripts.
:: TODO: Not supported yet
if "%TORCH_PACKAGE_NAME%" == "" set TORCH_PACKAGE_NAME=torchvision
:: PIP_UPLOAD_FOLDER should end in a slash. This is to handle it being empty
:: (when uploading to e.g. whl/cpu/) and also to handle nightlies (when
:: uploading to e.g. /whl/nightly/cpu)
:: TODO: Not supported yet
if "%PIP_UPLOAD_FOLDER%" == "" set "PIP_UPLOAD_FOLDER=nightly\"
:: The location of the binary_sizes dir in s3 is hardcoded into
:: upload_binary_sizes.sh
:: DAYS_TO_KEEP
:: How many days to keep around for clean.sh. Build folders older than this
:: will be purged at the end of cron jobs. '1' means to keep only the current
:: day. Values less than 1 are not allowed. The default is 5.
:: TODO: Not supported yet
if "%DAYS_TO_KEEP%" == "" set /a DAYS_TO_KEEP=5
if %DAYS_TO_KEEP% LSS 1 (
echo DAYS_TO_KEEP cannot be less than 1.
echo A value of 1 means to only keep the build for today
exit /b 1
)
@echo off
set SRC_DIR=%~dp0
pushd %SRC_DIR%
if NOT "%CUDA_VERSION%" == "cpu" (
set PACKAGE_SUFFIX=_cuda%CUDA_VERSION%
) else (
set PACKAGE_SUFFIX=
)
if "%PACKAGEFULLNAME%" == "Conda" (
set PACKAGE=conda
) else (
set PACKAGE=wheels
)
if not defined PACKAGE_SUFFIX (
set PUBLISH_BRANCH=vision_%PACKAGE%_%DESIRED_PYTHON%
) else (
set PUBLISH_BRANCH=vision_%PACKAGE%_%DESIRED_PYTHON%%PACKAGE_SUFFIX%
)
git clone %ARTIFACT_REPO_URL% -b %PUBLISH_BRANCH% --single-branch >nul 2>&1
IF ERRORLEVEL 1 (
echo Branch %PUBLISH_BRANCH% not exist, falling back to master
set NO_BRANCH=1
git clone %ARTIFACT_REPO_URL% -b master --single-branch >nul 2>&1
)
IF ERRORLEVEL 1 (
echo Clone failed
goto err
)
cd pytorch_builder
attrib -s -h -r . /s /d
:: Empty repo
rd /s /q . || ver >nul
IF NOT EXIST %PACKAGE% mkdir %PACKAGE%
xcopy /S /E /Y ..\..\output\*.* %PACKAGE%\
git config --global user.name "Azure DevOps"
git config --global user.email peterghost86@gmail.com
git init
git checkout --orphan %PUBLISH_BRANCH%
git remote add origin %ARTIFACT_REPO_URL%
git add .
git commit -m "Update artifacts"
:push
if "%RETRY_TIMES%" == "" (
set /a RETRY_TIMES=10
set /a SLEEP_TIME=2
) else (
set /a RETRY_TIMES=%RETRY_TIMES%-1
set /a SLEEP_TIME=%SLEEP_TIME%*2
)
git push origin %PUBLISH_BRANCH% -f > nul 2>&1
IF ERRORLEVEL 1 (
echo Git push retry times remaining: %RETRY_TIMES%
echo Sleep time: %SLEEP_TIME% seconds
IF %RETRY_TIMES% EQU 0 (
echo Push failed
goto err
)
waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
goto push
) ELSE (
set RETRY_TIMES=
set SLEEP_TIME=
)
popd
exit /b 0
:err
popd
exit /b 1
@echo off
echo The flags after configuring:
echo NO_CUDA=%NO_CUDA%
echo CMAKE_GENERATOR=%CMAKE_GENERATOR%
if "%NO_CUDA%"=="" echo CUDA_PATH=%CUDA_PATH%
if NOT "%CC%"=="" echo CC=%CC%
if NOT "%CXX%"=="" echo CXX=%CXX%
if NOT "%DISTUTILS_USE_SDK%"=="" echo DISTUTILS_USE_SDK=%DISTUTILS_USE_SDK%
set SRC_DIR=%~dp0\..
IF "%VSDEVCMD_ARGS%" == "" (
call "%VS15VCVARSALL%" x64
) ELSE (
call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS%
)
pushd %SRC_DIR%
IF NOT exist "setup.py" (
cd %MODULE_NAME%
)
if "%CXX%"=="sccache cl" (
sccache --stop-server
sccache --start-server
sccache --zero-stats
)
:pytorch
:: This stores in e.g. D:/_work/1/s/windows/output/cpu
pip wheel -e . --no-deps --wheel-dir ../output
:build_end
IF ERRORLEVEL 1 exit /b 1
IF NOT ERRORLEVEL 0 exit /b 1
if "%CXX%"=="sccache cl" (
taskkill /im sccache.exe /f /t || ver > nul
taskkill /im nvcc.exe /f /t || ver > nul
)
cd ..
@echo off
set SRC_DIR=%~dp0\..
pushd %SRC_DIR%
set PYTHON_VERSION=%PYTHON_PREFIX:py=cp%
if "%BUILD_VISION%" == "" (
pip install future pytest coverage hypothesis protobuf
) ELSE (
pip install future pytest "pillow>=4.1.1"
)
for /F "delims=" %%i in ('where /R %SRC_DIR%\output *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i"
if ERRORLEVEL 1 exit /b 1
if NOT "%BUILD_VISION%" == "" (
echo Smoke testing imports
python -c "import torchvision"
if ERRORLEVEL 1 exit /b 1
goto smoke_test_end
)
echo Smoke testing imports
python -c "import torch"
if ERRORLEVEL 1 exit /b 1
python -c "from caffe2.python import core"
if ERRORLEVEL 1 exit /b 1
echo Checking that MKL is available
python -c "import torch; exit(0 if torch.backends.mkl.is_available() else 1)"
if ERRORLEVEL 1 exit /b 1
setlocal EnableDelayedExpansion
set NVIDIA_GPU_EXISTS=0
for /F "delims=" %%i in ('wmic path win32_VideoController get name') do (
set GPUS=%%i
if not "x!GPUS:NVIDIA=!" == "x!GPUS!" (
SET NVIDIA_GPU_EXISTS=1
goto gpu_check_end
)
)
:gpu_check_end
endlocal & set NVIDIA_GPU_EXISTS=%NVIDIA_GPU_EXISTS%
if NOT "%CUDA_PREFIX%" == "cpu" if "%NVIDIA_GPU_EXISTS%" == "1" (
echo Checking that CUDA archs are setup correctly
python -c "import torch; torch.randn([3,5]).cuda()"
if ERRORLEVEL 1 exit /b 1
echo Checking that magma is available
python -c "import torch; torch.rand(1).cuda(); exit(0 if torch.cuda.has_magma else 1)"
if ERRORLEVEL 1 exit /b 1
echo Checking that CuDNN is available
python -c "import torch; exit(0 if torch.backends.cudnn.is_available() else 1)"
if ERRORLEVEL 1 exit /b 1
)
:smoke_test_end
echo Not running unit tests. Hopefully these problems are caught by CI
goto test_end
if "%BUILD_VISION%" == "" (
cd pytorch\test
python run_test.py -v
) else (
cd vision
pytest .
)
if ERRORLEVEL 1 exit /b 1
:test_end
popd
exit /b 0
@echo off
IF "%CONDA_UPLOADER_INSTALLATION%" == "" goto precheck_fail
IF "%PYTORCH_FINAL_PACKAGE_DIR%" == "" goto precheck_fail
IF "%today%" == "" goto precheck_fail
IF "%PYTORCH_ANACONDA_USERNAME%" == "" goto precheck_fail
IF "%PYTORCH_ANACONDA_PASSWORD%" == "" goto precheck_fail
goto precheck_pass
:precheck_fail
echo Please run nightly_defaults.bat first.
echo And remember to set `PYTORCH_FINAL_PACKAGE_DIR`
echo Finally, don't forget to set anaconda tokens
exit /b 1
:precheck_pass
pushd %today%
:: Install anaconda client
set "CONDA_HOME=%CONDA_UPLOADER_INSTALLATION%"
set "tmp_conda=%CONDA_HOME%"
set "miniconda_exe=%CD%\miniconda.exe"
rmdir /s /q "%CONDA_HOME%"
del miniconda.exe
curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%"
popd
IF ERRORLEVEL 1 (
echo Conda download failed
exit /b 1
)
call %~dp0\..\..\conda\install_conda.bat
IF ERRORLEVEL 1 (
echo Conda installation failed
exit /b 1
)
set "ORIG_PATH=%PATH%"
set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
REM conda install -y anaconda-client
pip install git+https://github.com/peterjc123/anaconda-client.git@log_more_meaningfull_errors
IF ERRORLEVEL 1 (
echo Anaconda client installation failed
exit /b 1
)
set PYTORCH_FINAL_PACKAGE=
:: Upload all the packages under `PYTORCH_FINAL_PACKAGE_DIR`
FOR /F "delims=" %%i IN ('where /R %PYTORCH_FINAL_PACKAGE_DIR% *vision*.tar.bz2') DO (
set "PYTORCH_FINAL_PACKAGE=%%i"
)
IF "%PYTORCH_FINAL_PACKAGE%" == "" (
echo No package to upload
exit /b 0
)
:upload
if "%RETRY_TIMES%" == "" (
set /a RETRY_TIMES=10
set /a SLEEP_TIME=2
) else (
set /a RETRY_TIMES=%RETRY_TIMES%-1
set /a SLEEP_TIME=%SLEEP_TIME%*2
)
REM bash -c "yes | anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%""
anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%"
IF ERRORLEVEL 1 (
echo Anaconda client login failed
exit /b 1
)
echo Uploading %PYTORCH_FINAL_PACKAGE% to Anaconda Cloud
anaconda upload "%PYTORCH_FINAL_PACKAGE%" -u pytorch-nightly --label main --force --no-progress
IF ERRORLEVEL 1 (
echo Anaconda upload retry times remaining: %RETRY_TIMES%
echo Sleep time: %SLEEP_TIME% seconds
IF %RETRY_TIMES% EQU 0 (
echo Upload failed
exit /b 1
)
waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
goto upload
) ELSE (
set RETRY_TIMES=
set SLEEP_TIME=
)
@echo off
set VS_DOWNLOAD_LINK=https://aka.ms/vs/15/release/vs_enterprise.exe
set VS_INSTALL_PATH=C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise
set VS_INSTALL_ARGS=--nocache --quiet --wait --add Microsoft.VisualStudio.Component.VC.Tools.14.11
set VSDEVCMD_ARGS=-vcvars_ver=14.11
curl -k -L %VS_DOWNLOAD_LINK% --output vs_installer.exe
if errorlevel 1 exit /b 1
start /wait vs_installer.exe modify --installPath "%VS_INSTALL_PATH%" %VS_INSTALL_ARGS%
if not errorlevel 0 exit /b 1
if errorlevel 1 if not errorlevel 3010 exit /b 1
if errorlevel 3011 exit /b 1
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
set NO_CUDA=
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
IF "%NVTOOLSEXT_PATH%"=="" (
echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
exit /b 1
goto optcheck
)
IF "%CUDA_PATH_V10_0%"=="" (
echo CUDA 10.0 not found, failing
exit /b 1
) ELSE (
IF "%BUILD_VISION%" == "" (
set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5
set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
) ELSE (
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50
)
set "CUDA_PATH=%CUDA_PATH_V10_0%"
set "PATH=%CUDA_PATH_V10_0%\bin;%PATH%"
)
:optcheck
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
set NO_CUDA=
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
IF "%NVTOOLSEXT_PATH%"=="" (
echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
exit /b 1
goto optcheck
)
IF "%CUDA_PATH_V9_0%"=="" (
echo CUDA 9 not found, failing
exit /b 1
) ELSE (
IF "%BUILD_VISION%" == "" (
set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;7.0
set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
) ELSE (
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
)
set "CUDA_PATH=%CUDA_PATH_V9_0%"
set "PATH=%CUDA_PATH_V9_0%\bin;%PATH%"
)
:optcheck
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
jobs:
- job: 'VSTS_Auth_Task'
timeoutInMinutes: 5
cancelTimeoutInMinutes: 5
variables:
- group: 'peterjc-vsts-token'
pool:
vmImage: 'vs2017-win2016'
steps:
- checkout: self
clean: true
- template: vsts_auth.yml
parameters:
auth: $(vsts_auth)
parameters:
msagent: false
steps:
- bash: 'find . -name "*.sh" -exec dos2unix {} +'
displayName: Replace file endings
- script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%'
displayName: 'Create final package directory'
- bash: './packaging/conda/build_vision.sh $CUDA_VERSION $TORCHVISION_BUILD_VERSION $TORCHVISION_BUILD_NUMBER'
displayName: Build
env:
${{ if eq(parameters.msagent, 'true') }}:
MAX_JOBS: 2
parameters:
package: ''
spec: ''
jobDesc: ''
packageDesc: ''
msagent: true
cpuEnabled: true
cudaEnabled: true
condaEnabled: true
wheelsEnabled: true
override: false
jobs:
- job: 'Windows_${{ parameters.spec }}_${{ parameters.package }}_Build'
timeoutInMinutes: 60
cancelTimeoutInMinutes: 5
condition: >
or(and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CPU'),
eq('${{ parameters.condaEnabled }}', 'true'), eq('${{ parameters.cpuEnabled }}', 'true')),
and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CPU'),
eq('${{ parameters.wheelsEnabled }}', 'true'), eq('${{ parameters.cpuEnabled }}', 'true')),
and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CUDA'),
eq('${{ parameters.condaEnabled }}', 'true'), eq('${{ parameters.cudaEnabled }}', 'true')),
and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CUDA'),
eq('${{ parameters.wheelsEnabled }}', 'true'), eq('${{ parameters.cudaEnabled }}', 'true')))
variables:
- ${{ if eq(parameters.override, 'true') }}:
- name: TORCHVISION_BUILD_NUMBER
value: 1
- name: PYTORCH_REPO
value: 'pytorch'
- name: PYTORCH_BRANCH
value: 'v0.4.0'
- ${{ if eq(parameters.msagent, 'true') }}:
- name: USE_SCCACHE
value: 0
- ${{ if eq(parameters.msagent, 'false') }}:
- name: USE_SCCACHE
value: 1
- ${{ if eq(parameters.package, 'Conda') }}:
- group: peterjc_anaconda_token
- name: PYTORCH_FINAL_PACKAGE_DIR
value: '$(Build.Repository.LocalPath)\packaging\windows\output'
strategy:
maxParallel: 10
matrix:
${{ if eq(parameters.spec, 'CPU') }}:
PY3.5:
DESIRED_PYTHON: 3.5
CUDA_VERSION: cpu
PY3.6:
DESIRED_PYTHON: 3.6
CUDA_VERSION: cpu
PY3.7:
DESIRED_PYTHON: 3.7
CUDA_VERSION: cpu
PY3.8:
DESIRED_PYTHON: 3.8
CUDA_VERSION: cpu
${{ if ne(parameters.spec, 'CPU') }}:
PY3.5_92:
DESIRED_PYTHON: 3.5
CUDA_VERSION: 92
PY3.6_92:
DESIRED_PYTHON: 3.6
CUDA_VERSION: 92
PY3.7_92:
DESIRED_PYTHON: 3.7
CUDA_VERSION: 92
PY3.8_92:
DESIRED_PYTHON: 3.8
CUDA_VERSION: 92
PY3.5_101:
DESIRED_PYTHON: 3.5
CUDA_VERSION: 101
PY3.6_101:
DESIRED_PYTHON: 3.6
CUDA_VERSION: 101
PY3.7_101:
DESIRED_PYTHON: 3.7
CUDA_VERSION: 101
PY3.8_101:
DESIRED_PYTHON: 3.8
CUDA_VERSION: 101
PY3.5_102:
DESIRED_PYTHON: 3.5
CUDA_VERSION: 102
PY3.6_102:
DESIRED_PYTHON: 3.6
CUDA_VERSION: 102
PY3.7_102:
DESIRED_PYTHON: 3.7
CUDA_VERSION: 102
PY3.8_102:
DESIRED_PYTHON: 3.8
CUDA_VERSION: 102
pool:
${{ if eq(parameters.msagent, 'true') }}:
vmImage: 'vs2017-win2016'
${{ if eq(parameters.msagent, 'false') }}:
name: 'release'
steps:
- checkout: self
clean: true
- template: setup_env_for_msagent.yml
parameters:
msagent: ${{ parameters.msagent }}
# - ${{ if and(eq(parameters.override, 'true'), eq(parameters.package, 'Wheels')) }}:
# - template: override_pytorch_version.yml
- template: setup_nightly_variables.yml
parameters:
package: ${{ parameters.package }}
- ${{ if eq(parameters.package, 'Wheels') }}:
- template: build_wheels.yml
parameters:
msagent: ${{ parameters.msagent }}
- ${{ if eq(parameters.package, 'Conda') }}:
- template: build_conda.yml
parameters:
msagent: ${{ parameters.msagent }}
- ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}:
- template: publish_test_results.yml
parameters:
msagent: ${{ parameters.msagent }}
# If you want to upload binaries to S3 & Anaconda Cloud, please uncomment this section.
- ${{ if and(eq(parameters.package, 'Wheels'), eq(parameters.spec, 'CPU')) }}:
- template: upload_to_s3.yml
parameters:
cuVer: '$(CUDA_VERSION)'
cudaVer: '$(CUDA_VERSION)'
- ${{ if and(eq(parameters.package, 'Wheels'), ne(parameters.spec, 'CPU')) }}:
- template: upload_to_s3.yml
parameters:
cuVer: 'cu$(CUDA_VERSION)'
cudaVer: 'cuda$(CUDA_VERSION)'
- ${{ if eq(parameters.package, 'Conda') }}:
- template: upload_to_conda.yml
parameters:
user: $(peterjc_conda_username)
pass: $(peterjc_conda_password)
# If you want to upload binaries to Azure Git, please uncomment this section.
# - ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}:
# - template: publish_test_results.yml
# parameters:
# msagent: ${{ parameters.msagent }}
# - template: publish_packages.yml
# parameters:
# package: ${{ parameters.package }}
parameters:
msagent: false
steps:
- script: 'call packaging/windows/build_vision.bat %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%'
displayName: Build
env:
${{ if eq(parameters.msagent, 'true') }}:
MAX_JOBS: 2
parameters:
msagent: true
enabled: false
jobs:
- job: 'Linux_CPU_Conda_Build'
timeoutInMinutes: 0
cancelTimeoutInMinutes: 5
condition: ${{ eq(parameters.enabled, 'true') }}
variables:
CUDA_VERSION: cpu
TORCH_CONDA_BUILD_FOLDER: pytorch-nightly
PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output'
strategy:
maxParallel: 10
matrix:
PY3.5:
DESIRED_PYTHON: 3.5
pool:
vmImage: 'ubuntu-16.04'
steps:
- checkout: self
clean: true
- script: 'sudo apt-get install p7zip-full'
displayName: 'Install 7Zip'
- task: CondaEnvironment@1
displayName: 'Install conda-build'
inputs:
packageSpecs: 'conda-build'
- template: build_conda.yml
parameters:
msagent: ${{ parameters.msagent }}
steps:
- script: 'windows/internal/override_pytorch_version.bat'
displayName: 'Override PyTorch Build Version for Wheels'
- script: 'echo $(PYTORCH_BUILD_VERSION)'
displayName: 'Show PyTorch Build Version'
parameters:
package: ''
steps:
- script: 'packaging/windows/internal/publish.bat'
displayName: 'Upload packages to Azure DevOps Repo'
env:
PACKAGEFULLNAME: ${{ parameters.package }}
steps:
- task: PublishTestResults@2 # No test results to publish
inputs:
testResultsFiles: 'windows/pytorch/test/**/*.xml'
testRunTitle: 'Publish test results'
enabled: false
parameters:
msagent: false
steps:
- ${{ if eq(parameters.msagent, 'true') }}:
- task: BatchScript@1
displayName: 'Install 7Zip & cURL'
inputs:
filename: 'packaging/windows/internal/dep_install.bat'
modifyEnvironment: true
- task: BatchScript@1
displayName: 'Install Visual Studio 2017'
inputs:
filename: 'packaging/windows/internal/vs_install.bat'
modifyEnvironment: true
- task: BatchScript@1
displayName: 'Install CUDA'
inputs:
filename: 'packaging/windows/internal/cuda_install.bat'
modifyEnvironment: true
parameters:
package: ''
steps:
- task: BatchScript@1
displayName: 'Setup nightly variables'
inputs:
filename: 'packaging/windows/internal/nightly_defaults.bat'
arguments: ${{ parameters.package }}
modifyEnvironment: true
parameters:
user: ''
pass: ''
steps:
- script: 'call packaging/windows/internal/upload.bat'
displayName: 'Upload packages to Anaconda Cloud'
env:
PYTORCH_ANACONDA_USERNAME: ${{ parameters.user }}
PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment