Commit 0fc002df authored by huchen's avatar huchen
Browse files

init the dlexamples new

parent 0e04b692
{% set vcver="14.2" %}
{% set vcfeature="14" %}
{% set vsyear="2019" %}
{% set fullver="15.4.27004.2010" %}
package:
name: vs{{ vsyear }}
version: {{ fullver }}
build:
skip: True [not win]
script_env:
- VSDEVCMD_ARGS # [win]
outputs:
- name: vs{{ vsyear }}_{{ cross_compiler_target_platform }}
script: install_activate.bat
track_features:
# VS 2019 is binary-compatible with VS 2017/vc 14.1 and 2015/vc14. Tools are "v142".
strong:
- vc{{ vcfeature }}
about:
summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler
license: BSD 3-clause
#!/bin/bash
set -ex
if [ "$#" -ne 1 ]; then
echo "Illegal number of parameters. Pass cuda version"
echo "CUDA version should be cu92, cu100 or cpu"
exit 1
fi
export CUVER="$1" # cu[0-9]* cpu
if [[ "$CUVER" == "cu102" ]]; then
cu_suffix=""
else
cu_suffix="+$CUVER"
fi
export TORCHVISION_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")${cu_suffix}"
export TORCHVISION_BUILD_NUMBER="1"
export TORCHVISION_LOCAL_VERSION_LABEL="$CUVER"
export OUT_DIR="/remote/$CUVER"
pushd /opt/python
DESIRED_PYTHON=(*/)
popd
for desired_py in "${DESIRED_PYTHON[@]}"; do
python_installations+=("/opt/python/$desired_py")
done
OLD_PATH=$PATH
cd /tmp
rm -rf vision
git clone https://github.com/pytorch/vision
cd /tmp/vision
for PYDIR in "${python_installations[@]}"; do
export PATH=$PYDIR/bin:$OLD_PATH
pip install --upgrade pip
pip install numpy pyyaml future
pip uninstall -y torch || true
pip uninstall -y torch_nightly || true
export TORCHVISION_PYTORCH_DEPENDENCY_NAME=torch_nightly
pip install torch_nightly -f https://download.pytorch.org/whl/nightly/$CUVER/torch_nightly.html
# CPU/CUDA variants of PyTorch have ABI compatible PyTorch for
# the CPU only bits. Therefore, we
# strip off the local package qualifier, but ONLY if we're
# doing a CPU build.
if [[ "$CUVER" == "cpu" ]]; then
export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')"
else
export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//')"
fi
echo "Building against ${TORCHVISION_PYTORCH_DEPENDENCY_VERSION}"
pip install ninja
python setup.py clean
python setup.py bdist_wheel
mkdir -p $OUT_DIR
cp dist/*.whl $OUT_DIR/
done
if [[ ":$PATH:" == *"conda"* ]]; then
echo "existing anaconda install in PATH, remove it and run script"
exit 1
fi
# download and activate anaconda
rm -rf ~/minconda_wheel_env_tmp
wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \
chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \
./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \
rm Miniconda3-latest-MacOSX-x86_64.sh
. ~/minconda_wheel_env_tmp/bin/activate
export TORCHVISION_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")"
export TORCHVISION_BUILD_NUMBER="1"
export OUT_DIR=~/torchvision_wheels
export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++
pushd /tmp
rm -rf vision
git clone https://github.com/pytorch/vision
pushd vision
desired_pythons=( "2.7" "3.5" "3.6" "3.7" )
# for each python
for desired_python in "${desired_pythons[@]}"
do
# create and activate python env
env_name="env$desired_python"
conda create -yn $env_name python="$desired_python"
conda activate $env_name
pip uninstall -y torch || true
pip uninstall -y torch_nightly || true
export TORCHVISION_PYTORCH_DEPENDENCY_NAME=torch_nightly
pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: *//')"
echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}"
# install torchvision dependencies
pip install ninja scipy pytest
python setup.py clean
python setup.py bdist_wheel
mkdir -p $OUT_DIR
cp dist/*.whl $OUT_DIR/
done
popd
popd
# -*- coding: utf-8 -*-
"""Helper script to package wheels and relocate binaries."""
# Standard library imports
import os
import io
import sys
import glob
import shutil
import zipfile
import hashlib
import platform
import subprocess
import os.path as osp
from base64 import urlsafe_b64encode
# Third party imports
if sys.platform == 'linux':
from auditwheel.lddtree import lddtree
from wheel.bdist_wheel import get_abi_tag
WHITELIST = {
'libgcc_s.so.1', 'libstdc++.so.6', 'libm.so.6',
'libdl.so.2', 'librt.so.1', 'libc.so.6',
'libnsl.so.1', 'libutil.so.1', 'libpthread.so.0',
'libresolv.so.2', 'libX11.so.6', 'libXext.so.6',
'libXrender.so.1', 'libICE.so.6', 'libSM.so.6',
'libGL.so.1', 'libgobject-2.0.so.0', 'libgthread-2.0.so.0',
'libglib-2.0.so.0', 'ld-linux-x86-64.so.2', 'ld-2.17.so'
}
WINDOWS_WHITELIST = {
'MSVCP140.dll', 'KERNEL32.dll',
'VCRUNTIME140_1.dll', 'VCRUNTIME140.dll',
'api-ms-win-crt-heap-l1-1-0.dll',
'api-ms-win-crt-runtime-l1-1-0.dll',
'api-ms-win-crt-stdio-l1-1-0.dll',
'api-ms-win-crt-filesystem-l1-1-0.dll',
'api-ms-win-crt-string-l1-1-0.dll',
'api-ms-win-crt-environment-l1-1-0.dll',
'api-ms-win-crt-math-l1-1-0.dll',
'api-ms-win-crt-convert-l1-1-0.dll'
}
HERE = osp.dirname(osp.abspath(__file__))
PACKAGE_ROOT = osp.dirname(osp.dirname(HERE))
PLATFORM_ARCH = platform.machine()
PYTHON_VERSION = sys.version_info
def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
"""Yield pieces of data from a file-like object until EOF."""
while True:
chunk = file.read(size)
if not chunk:
break
yield chunk
def rehash(path, blocksize=1 << 20):
"""Return (hash, length) for path using hashlib.sha256()"""
h = hashlib.sha256()
length = 0
with open(path, 'rb') as f:
for block in read_chunks(f, size=blocksize):
length += len(block)
h.update(block)
digest = 'sha256=' + urlsafe_b64encode(
h.digest()
).decode('latin1').rstrip('=')
# unicode/str python2 issues
return (digest, str(length)) # type: ignore
def unzip_file(file, dest):
"""Decompress zip `file` into directory `dest`."""
with zipfile.ZipFile(file, 'r') as zip_ref:
zip_ref.extractall(dest)
def is_program_installed(basename):
"""
Return program absolute path if installed in PATH.
Otherwise, return None
On macOS systems, a .app is considered installed if
it exists.
"""
if (sys.platform == 'darwin' and basename.endswith('.app') and
osp.exists(basename)):
return basename
for path in os.environ["PATH"].split(os.pathsep):
abspath = osp.join(path, basename)
if osp.isfile(abspath):
return abspath
def find_program(basename):
"""
Find program in PATH and return absolute path
Try adding .exe or .bat to basename on Windows platforms
(return None if not found)
"""
names = [basename]
if os.name == 'nt':
# Windows platforms
extensions = ('.exe', '.bat', '.cmd', '.dll')
if not basename.endswith(extensions):
names = [basename + ext for ext in extensions] + [basename]
for name in names:
path = is_program_installed(name)
if path:
return path
def patch_new_path(library_path, new_dir):
library = osp.basename(library_path)
name, *rest = library.split('.')
rest = '.'.join(rest)
hash_id = hashlib.sha256(library_path.encode('utf-8')).hexdigest()[:8]
new_name = '.'.join([name, hash_id, rest])
return osp.join(new_dir, new_name)
def find_dll_dependencies(dumpbin, binary):
out = subprocess.run([dumpbin, "/dependents", binary],
stdout=subprocess.PIPE)
out = out.stdout.strip().decode('utf-8')
start_index = out.find('dependencies:') + len('dependencies:')
end_index = out.find('Summary')
dlls = out[start_index:end_index].strip()
dlls = dlls.split(os.linesep)
dlls = [dll.strip() for dll in dlls]
return dlls
def relocate_elf_library(patchelf, output_dir, output_library, binary):
"""
Relocate an ELF shared library to be packaged on a wheel.
Given a shared library, find the transitive closure of its dependencies,
rename and copy them into the wheel while updating their respective rpaths.
"""
print('Relocating {0}'.format(binary))
binary_path = osp.join(output_library, binary)
ld_tree = lddtree(binary_path)
tree_libs = ld_tree['libs']
binary_queue = [(n, binary) for n in ld_tree['needed']]
binary_paths = {binary: binary_path}
binary_dependencies = {}
while binary_queue != []:
library, parent = binary_queue.pop(0)
library_info = tree_libs[library]
print(library)
if library_info['path'] is None:
print('Omitting {0}'.format(library))
continue
if library in WHITELIST:
# Omit glibc/gcc/system libraries
print('Omitting {0}'.format(library))
continue
parent_dependencies = binary_dependencies.get(parent, [])
parent_dependencies.append(library)
binary_dependencies[parent] = parent_dependencies
if library in binary_paths:
continue
binary_paths[library] = library_info['path']
binary_queue += [(n, library) for n in library_info['needed']]
print('Copying dependencies to wheel directory')
new_libraries_path = osp.join(output_dir, 'torchvision.libs')
os.makedirs(new_libraries_path)
new_names = {binary: binary_path}
for library in binary_paths:
if library != binary:
library_path = binary_paths[library]
new_library_path = patch_new_path(library_path, new_libraries_path)
print('{0} -> {1}'.format(library, new_library_path))
shutil.copyfile(library_path, new_library_path)
new_names[library] = new_library_path
print('Updating dependency names by new files')
for library in binary_paths:
if library != binary:
if library not in binary_dependencies:
continue
library_dependencies = binary_dependencies[library]
new_library_name = new_names[library]
for dep in library_dependencies:
new_dep = osp.basename(new_names[dep])
print('{0}: {1} -> {2}'.format(library, dep, new_dep))
subprocess.check_output(
[
patchelf,
'--replace-needed',
dep,
new_dep,
new_library_name
],
cwd=new_libraries_path)
print('Updating library rpath')
subprocess.check_output(
[
patchelf,
'--set-rpath',
"$ORIGIN",
new_library_name
],
cwd=new_libraries_path)
subprocess.check_output(
[
patchelf,
'--print-rpath',
new_library_name
],
cwd=new_libraries_path)
print("Update library dependencies")
library_dependencies = binary_dependencies[binary]
for dep in library_dependencies:
new_dep = osp.basename(new_names[dep])
print('{0}: {1} -> {2}'.format(binary, dep, new_dep))
subprocess.check_output(
[
patchelf,
'--replace-needed',
dep,
new_dep,
binary
],
cwd=output_library)
print('Update library rpath')
subprocess.check_output(
[
patchelf,
'--set-rpath',
"$ORIGIN:$ORIGIN/../torchvision.libs",
binary_path
],
cwd=output_library
)
def relocate_dll_library(dumpbin, output_dir, output_library, binary):
"""
Relocate a DLL/PE shared library to be packaged on a wheel.
Given a shared library, find the transitive closure of its dependencies,
rename and copy them into the wheel.
"""
print('Relocating {0}'.format(binary))
binary_path = osp.join(output_library, binary)
library_dlls = find_dll_dependencies(dumpbin, binary_path)
binary_queue = [(dll, binary) for dll in library_dlls]
binary_paths = {binary: binary_path}
binary_dependencies = {}
while binary_queue != []:
library, parent = binary_queue.pop(0)
if library in WINDOWS_WHITELIST or library.startswith('api-ms-win'):
print('Omitting {0}'.format(library))
continue
library_path = find_program(library)
if library_path is None:
print('{0} not found'.format(library))
continue
if osp.basename(osp.dirname(library_path)) == 'system32':
continue
print('{0}: {1}'.format(library, library_path))
parent_dependencies = binary_dependencies.get(parent, [])
parent_dependencies.append(library)
binary_dependencies[parent] = parent_dependencies
if library in binary_paths:
continue
binary_paths[library] = library_path
downstream_dlls = find_dll_dependencies(dumpbin, library_path)
binary_queue += [(n, library) for n in downstream_dlls]
print('Copying dependencies to wheel directory')
package_dir = osp.join(output_dir, 'torchvision')
for library in binary_paths:
if library != binary:
library_path = binary_paths[library]
new_library_path = osp.join(package_dir, library)
print('{0} -> {1}'.format(library, new_library_path))
shutil.copyfile(library_path, new_library_path)
def compress_wheel(output_dir, wheel, wheel_dir, wheel_name):
"""Create RECORD file and compress wheel distribution."""
print('Update RECORD file in wheel')
dist_info = glob.glob(osp.join(output_dir, '*.dist-info'))[0]
record_file = osp.join(dist_info, 'RECORD')
with open(record_file, 'w') as f:
for root, _, files in os.walk(output_dir):
for this_file in files:
full_file = osp.join(root, this_file)
rel_file = osp.relpath(full_file, output_dir)
if full_file == record_file:
f.write('{0},,\n'.format(rel_file))
else:
digest, size = rehash(full_file)
f.write('{0},{1},{2}\n'.format(rel_file, digest, size))
print('Compressing wheel')
base_wheel_name = osp.join(wheel_dir, wheel_name)
shutil.make_archive(base_wheel_name, 'zip', output_dir)
os.remove(wheel)
shutil.move('{0}.zip'.format(base_wheel_name), wheel)
shutil.rmtree(output_dir)
def patch_linux():
# Get patchelf location
patchelf = find_program('patchelf')
if patchelf is None:
raise FileNotFoundError('Patchelf was not found in the system, please'
' make sure that is available on the PATH.')
# Find wheel
print('Finding wheels...')
wheels = glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl'))
output_dir = osp.join(PACKAGE_ROOT, 'dist', '.wheel-process')
image_binary = 'image.so'
video_binary = 'video_reader.so'
torchvision_binaries = [image_binary, video_binary]
for wheel in wheels:
if osp.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print('Unzipping wheel...')
wheel_file = osp.basename(wheel)
wheel_dir = osp.dirname(wheel)
print('{0}'.format(wheel_file))
wheel_name, _ = osp.splitext(wheel_file)
unzip_file(wheel, output_dir)
print('Finding ELF dependencies...')
output_library = osp.join(output_dir, 'torchvision')
for binary in torchvision_binaries:
if osp.exists(osp.join(output_library, binary)):
relocate_elf_library(
patchelf, output_dir, output_library, binary)
compress_wheel(output_dir, wheel, wheel_dir, wheel_name)
def patch_win():
# Get dumpbin location
dumpbin = find_program('dumpbin')
if dumpbin is None:
raise FileNotFoundError('Dumpbin was not found in the system, please'
' make sure that is available on the PATH.')
# Find wheel
print('Finding wheels...')
wheels = glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl'))
output_dir = osp.join(PACKAGE_ROOT, 'dist', '.wheel-process')
image_binary = 'image.pyd'
video_binary = 'video_reader.pyd'
torchvision_binaries = [image_binary, video_binary]
for wheel in wheels:
if osp.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print('Unzipping wheel...')
wheel_file = osp.basename(wheel)
wheel_dir = osp.dirname(wheel)
print('{0}'.format(wheel_file))
wheel_name, _ = osp.splitext(wheel_file)
unzip_file(wheel, output_dir)
print('Finding DLL/PE dependencies...')
output_library = osp.join(output_dir, 'torchvision')
for binary in torchvision_binaries:
if osp.exists(osp.join(output_library, binary)):
relocate_dll_library(
dumpbin, output_dir, output_library, binary)
compress_wheel(output_dir, wheel, wheel_dir, wheel_name)
if __name__ == '__main__':
if sys.platform == 'linux':
patch_linux()
elif sys.platform == 'win32':
patch_win()
# Turn off auto builds for commits
trigger: none
pr: none
jobs:
- template: templates/build_task.yml
parameters:
package: 'Wheels'
spec: 'CPU'
msagent: true
# Turn off auto builds for commits
trigger: none
pr: none
jobs:
- template: templates/auth_task.yml
- template: templates/build_task.yml
parameters:
package: 'Wheels'
spec: 'CPU'
msagent: true
- template: templates/build_task.yml
parameters:
package: 'Conda'
spec: 'CPU'
msagent: true
- template: templates/build_task.yml
parameters:
package: 'Wheels'
spec: 'CUDA'
msagent: true
- template: templates/build_task.yml
parameters:
package: 'Conda'
spec: 'CUDA'
msagent: true
- template: templates/linux_build_task.yml
parameters:
msagent: $(ms.hosted.agent.cpu)
@echo off
:: This script parses args, installs required libraries (miniconda, MKL,
:: Magma), and then delegates to cpu.bat, cuda80.bat, etc.
IF NOT "%CUDA_VERSION%" == "" IF NOT "%TORCHVISION_BUILD_VERSION%" == "" if NOT "%TORCHVISION_BUILD_NUMBER%" == "" goto env_end
if "%~1"=="" goto arg_error
if "%~2"=="" goto arg_error
if "%~3"=="" goto arg_error
if NOT "%~4"=="" goto arg_error
goto arg_end
:arg_error
echo Illegal number of parameters. Pass cuda version, pytorch version, build number
echo CUDA version should be Mm with no dot, e.g. '80'
echo DESIRED_PYTHON should be M.m, e.g. '2.7'
exit /b 1
:arg_end
set CUDA_VERSION=%~1
set TORCHVISION_BUILD_VERSION=%~2
set TORCHVISION_BUILD_NUMBER=%~3
set BUILD_VERSION=%TORCHVISION_BUILD_VERSION%
:env_end
if NOT "%CUDA_VERSION%" == "cpu" (
set CUDA_PREFIX=cuda%CUDA_VERSION%
set CUVER=cu%CUDA_VERSION%
set FORCE_CUDA=1
) else (
set CUDA_PREFIX=cpu
set CUVER=cpu
)
set BUILD_VISION=1
REM set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index
IF "%DESIRED_PYTHON%" == "" set DESIRED_PYTHON=3.5;3.6;3.7
set DESIRED_PYTHON_PREFIX=%DESIRED_PYTHON:.=%
set DESIRED_PYTHON_PREFIX=py%DESIRED_PYTHON_PREFIX:;=;py%
set SRC_DIR=%~dp0
pushd %SRC_DIR%
:: Install Miniconda3
set "CONDA_HOME=%CD%\conda"
set "tmp_conda=%CONDA_HOME%"
set "miniconda_exe=%CD%\miniconda.exe"
rmdir /s /q conda
del miniconda.exe
curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%"
call ..\conda\install_conda.bat
IF ERRORLEVEL 1 exit /b 1
set "ORIG_PATH=%PATH%"
set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
:: Create a new conda environment
setlocal EnableDelayedExpansion
FOR %%v IN (%DESIRED_PYTHON%) DO (
set PYTHON_VERSION_STR=%%v
set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
conda remove -n py!PYTHON_VERSION_STR! --all -y || rmdir %CONDA_HOME%\envs\py!PYTHON_VERSION_STR! /s
conda create -n py!PYTHON_VERSION_STR! -y -q -c defaults -c conda-forge numpy>=1.11 mkl>=2018 python=%%v ca-certificates scipy
)
:: Uncomment for stable releases
:: FOR %%v IN (%DESIRED_PYTHON%) DO (
:: set PYTHON_VERSION_STR=%%v
:: set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
:: set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%"
:: if "%CUDA_VERSION%" == "100" (
:: set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl
:: ) else (
:: set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0%%2B%CUVER%-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl
:: )
:: echo Installing !TORCH_WHEEL!...
:: pip install "!TORCH_WHEEL!"
:: )
:: Uncomment for nightly releases
FOR %%v IN (%DESIRED_PYTHON%) DO (
set PYTHON_VERSION_STR=%%v
set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%"
set TORCH_WHEEL=torch --pre -f https://download.pytorch.org/whl/nightly/%CUVER%/torch_nightly.html
echo Installing !TORCH_WHEEL!...
pip install !TORCH_WHEEL!
)
endlocal
if "%DEBUG%" == "1" (
set BUILD_TYPE=debug
) ELSE (
set BUILD_TYPE=release
)
:: Install sccache
if "%USE_SCCACHE%" == "1" (
mkdir %CD%\tmp_bin
curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %CD%\tmp_bin\sccache.exe
if not "%CUDA_VERSION%" == "" (
copy %CD%\tmp_bin\sccache.exe %CD%\tmp_bin\nvcc.exe
set CUDA_NVCC_EXECUTABLE=%CD%\tmp_bin\nvcc
set "PATH=%CD%\tmp_bin;%PATH%"
)
)
for %%v in (%DESIRED_PYTHON_PREFIX%) do (
:: Activate Python Environment
set PYTHON_PREFIX=%%v
set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%"
if defined INCLUDE (
set "INCLUDE=%INCLUDE%;%CONDA_HOME%\envs\%%v\Library\include"
) else (
set "INCLUDE=%CONDA_HOME%\envs\%%v\Library\include"
)
if defined LIB (
set "LIB=%LIB%;%CONDA_HOME%\envs\%%v\Library\lib"
) else (
set "LIB=%CONDA_HOME%\envs\%%v\Library\lib"
)
@setlocal
:: Set Flags
if NOT "%CUDA_VERSION%"=="cpu" (
set CUDNN_VERSION=7
)
call %CUDA_PREFIX%.bat
IF ERRORLEVEL 1 exit /b 1
call internal\test.bat
IF ERRORLEVEL 1 exit /b 1
@endlocal
)
set "PATH=%ORIG_PATH%"
popd
IF ERRORLEVEL 1 exit /b 1
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
echo Disabling CUDA
set NO_CUDA=1
set USE_CUDA=0
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy_cpu.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
set NO_CUDA=
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
IF "%NVTOOLSEXT_PATH%"=="" (
echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
exit /b 1
goto optcheck
)
IF "%CUDA_PATH_V10_1%"=="" (
echo CUDA 10.1 not found, failing
exit /b 1
) ELSE (
IF "%BUILD_VISION%" == "" (
set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5
set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
) ELSE (
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50
)
set "CUDA_PATH=%CUDA_PATH_V10_1%"
set "PATH=%CUDA_PATH_V10_1%\bin;%PATH%"
)
:optcheck
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
set NO_CUDA=
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
IF "%NVTOOLSEXT_PATH%"=="" (
echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
exit /b 1
goto optcheck
)
IF "%CUDA_PATH_V10_2%"=="" (
echo CUDA 10.2 not found, failing
exit /b 1
) ELSE (
IF "%BUILD_VISION%" == "" (
set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5
set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
) ELSE (
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50
)
set "CUDA_PATH=%CUDA_PATH_V10_2%"
set "PATH=%CUDA_PATH_V10_2%\bin;%PATH%"
)
:optcheck
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
set USE_CUDA=
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
IF "%NVTOOLSEXT_PATH%"=="" (
echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
exit /b 1
goto optcheck
)
IF "%CUDA_PATH_V9_2%"=="" (
echo CUDA 9.2 not found, failing
exit /b 1
) ELSE (
IF "%BUILD_VISION%" == "" (
set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0
set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
) ELSE (
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
)
set "CUDA_PATH=%CUDA_PATH_V9_2%"
set "PATH=%CUDA_PATH_V9_2%\bin;%PATH%"
)
:optcheck
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
@echo off
: From the following doc, the build won't be triggered if the users don't sign in daily.
: https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?tabs=yaml&view=vsts#my-build-didnt-run-what-happened
: To avoid this problem, we can just go through the sign in process using the following command.
:auth_start
if "%RETRY_TIMES%" == "" (
set /a RETRY_TIMES=10
set /a SLEEP_TIME=2
) else (
set /a RETRY_TIMES=%RETRY_TIMES%-1
set /a SLEEP_TIME=%SLEEP_TIME%*2
)
for /f "usebackq tokens=*" %%i in (`curl -so NUL -w "%%{http_code}" -u %VSTS_AUTH% https://dev.azure.com/pytorch`) do (
set STATUS_CODE=%%i
)
IF NOT "%STATUS_CODE%" == "200" (
echo Auth retry times remaining: %RETRY_TIMES%
echo Sleep time: %SLEEP_TIME% seconds
IF %RETRY_TIMES% EQU 0 (
echo Auth failed
goto err
)
waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
goto auth_start
) ELSE (
echo Login Attempt Succeeded
goto auth_end
)
:err
: Throw a warning if it fails
powershell -c "Write-Warning 'Login Attempt Failed'"
:auth_end
set RETRY_TIMES=
set SLEEP_TIME=
set STATUS_CODE=
exit /b 0
@echo on
msbuild "-p:Configuration=Release" torchvision.vcxproj
msbuild "-p:Configuration=Release" INSTALL.vcxproj
if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.13
if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1
if errorlevel 1 exit /b 1
call packaging/windows/internal/cuda_install.bat
if errorlevel 1 exit /b 1
call packaging/windows/internal/nightly_defaults.bat Conda
if errorlevel 1 exit /b 1
set PYTORCH_FINAL_PACKAGE_DIR=%CD%\packaging\windows\output
if not exist "%PYTORCH_FINAL_PACKAGE_DIR%" mkdir %PYTORCH_FINAL_PACKAGE_DIR%
bash ./packaging/conda/build_vision.sh %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%
if errorlevel 1 exit /b 1
@echo on
set CL=/I"C:\Program Files (x86)\torchvision\include"
msbuild "-p:Configuration=Release" hello-world.vcxproj
@echo on
set CL=/I"C:\Program Files (x86)\torchvision\include"
msbuild "-p:Configuration=Release" test_frcnn_tracing.vcxproj
if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.13
if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1
if errorlevel 1 exit /b 1
call packaging/windows/internal/cuda_install.bat
if errorlevel 1 exit /b 1
call packaging/windows/internal/nightly_defaults.bat Wheels
if errorlevel 1 exit /b 1
call packaging/windows/build_vision.bat %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%
if errorlevel 1 exit /b 1
@echo off
REM Check for necessary components
IF NOT "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
echo You should use 64 bits Windows to build and run PyTorch
exit /b 1
)
IF "%BUILD_VISION%" == "" (
where /q cmake.exe
IF ERRORLEVEL 1 (
echo CMake is required to compile PyTorch on Windows
exit /b 1
)
)
IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" (
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
exit /b 1
)
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VS15INSTALLDIR=%%i"
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
goto vswhere
)
)
:vswhere
IF "%VS15VCVARSALL%"=="" (
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
exit /b 1
)
set MSSdk=1
set DISTUTILS_USE_SDK=1
where /q python.exe
IF ERRORLEVEL 1 (
echo Python x64 3.5 or up is required to compile PyTorch on Windows
exit /b 1
)
for /F "usebackq delims=" %%i in (`python -c "import sys; print('{0[0]}{0[1]}'.format(sys.version_info))"`) do (
set /a PYVER=%%i
)
if %PYVER% LSS 35 (
echo Warning: PyTorch for Python 2 under Windows is experimental.
echo Python x64 3.5 or up is recommended to compile PyTorch on Windows
echo Maybe you can create a virual environment if you have conda installed:
echo ^> conda create -n test python=3.6 pyyaml mkl numpy
echo ^> activate test
)
for /F "usebackq delims=" %%i in (`python -c "import struct;print( 8 * struct.calcsize('P'))"`) do (
set /a PYSIZE=%%i
)
if %PYSIZE% NEQ 64 (
echo Python x64 3.5 or up is required to compile PyTorch on Windows
exit /b 1
)
@echo off
REM Check for optional components
where /q ninja.exe
IF NOT ERRORLEVEL 1 (
echo Ninja found, using it to speed up builds
set CMAKE_GENERATOR=Ninja
)
where /q clcache.exe
IF NOT ERRORLEVEL 1 (
echo clcache found, using it to speed up builds
set CC=clcache
set CXX=clcache
)
where /q sccache.exe
IF NOT ERRORLEVEL 1 (
echo sccache found, using it to speed up builds
set CC=sccache cl
set CXX=sccache cl
)
IF exist "%MKLProductDir%\mkl\lib\intel64_win" (
echo MKL found, adding it to build
set "LIB=%MKLProductDir%\mkl\lib\intel64_win;%MKLProductDir%\compiler\lib\intel64_win;%LIB%";
)
exit /b 0
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment