add packaging scripts (#996)

packaging/README.md

+# Building torchvision packages for release
+
+## Anaconda packages
+
+### Linux
+
+```bash
+nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash
+pushd remote/conda
+
+./build_vision.sh 9.0
+./build_vision.sh 10.0
+./build_vision.sh cpu
+
+# copy packages over to /remote
+# exit docker
+# anaconda upload -u pytorch torchvision*.bz2
+```
+
+### OSX
+
+```bash
+# create a fresh anaconda environment / install and activate it
+conda install -y conda-build anaconda-client
+./build_vision.sh cpu
+
+# copy packages over to /remote
+# exit docker
+# anaconda upload -u pytorch torchvision*.bz2
+```
+
+## Wheels
+
+### Linux
+
+pushd wheel
+
+```bash
+nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash
+cd remote
+./linux_manywheel.sh cu90
+
+rm -rf /usr/local/cuda*
+./linux_manywheel.sh cpu
+```
+
+```bash
+nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda100:latest bash
+cd remote
+./linux_manywheel.sh cu100
+```
+
+wheels are in the folders `cpu`, `cu90`, `cu100`.
+
+You can upload the `cu90` wheels to twine with `twine upload *.whl`.
+Which wheels we upload depends on which wheels PyTorch uploads as default, and right now, it's `cu90`.
+
+### OSX
+
+```bash
+pushd wheel
+./osx_wheel.sh
+```
+
+wheels are in the current folder.
+
+You can upload them to twine with `twine upload *.whl`

packaging/conda/build_vision.sh

+#!/usr/bin/env bash
+if [[ -x "/remote/anaconda_token" ]]; then
+    . /remote/anaconda_token || true
+fi
+
+set -ex
+
+# Function to retry functions that sometimes timeout or have flaky failures
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+if [ "$#" -ne 1 ]; then
+    echo "Illegal number of parameters. Pass cuda version"
+    echo "CUDA version should be M.m with no dot, e.g. '8.0' or 'cpu'"
+    exit 1
+fi
+desired_cuda="$1"
+
+export TORCHVISION_BUILD_VERSION="0.3.0"
+export TORCHVISION_BUILD_NUMBER=1
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+if [[ -z "$WIN_PACKAGE_WORK_DIR" ]]; then
+    WIN_PACKAGE_WORK_DIR="$(echo $(pwd -W) | tr '/' '\\')\\tmp_conda_$(date +%H%M%S)"
+fi
+
+if [[ "$OSTYPE" == "msys" ]]; then
+    mkdir -p "$WIN_PACKAGE_WORK_DIR" || true
+    vision_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchvision-src"
+    git config --system core.longpaths true
+else
+    vision_rootdir="$(pwd)/torchvision-src"
+fi
+
+if [[ ! -d "$vision_rootdir" ]]; then
+    rm -rf "$vision_rootdir"
+    git clone "https://github.com/pytorch/vision" "$vision_rootdir"
+    pushd "$vision_rootdir"
+    git checkout v$TORCHVISION_BUILD_VERSION
+    popd
+fi
+
+cd "$SOURCE_DIR"
+
+if [[ "$OSTYPE" == "msys" ]]; then
+    export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda"
+    export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe"
+    rm -rf "$tmp_conda"
+    rm -f "$miniconda_exe"
+    curl -sSk https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe"
+    "$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe"
+    pushd $tmp_conda
+    export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH"
+    popd
+    # We have to skip 3.17 because of the following bug.
+    # https://github.com/conda/conda-build/issues/3285
+    retry conda install -yq conda-build
+fi
+
+ANACONDA_USER=pytorch
+conda config --set anaconda_upload no
+
+
+export TORCHVISION_PACKAGE_SUFFIX=""
+if [[ "$desired_cuda" == 'cpu' ]]; then
+    export CONDA_CUDATOOLKIT_CONSTRAINT=""
+    export CUDA_VERSION="None"
+    if [[ "$OSTYPE" != "darwin"* ]]; then
+        export TORCHVISION_PACKAGE_SUFFIX="-cpu"
+    fi
+else
+    . ./switch_cuda_version.sh $desired_cuda
+    if [[ "$desired_cuda" == "10.0" ]]; then
+	export CONDA_CUDATOOLKIT_CONSTRAINT="    - cudatoolkit >=10.0,<10.1 # [not osx]"
+    elif [[ "$desired_cuda" == "9.0" ]]; then
+	export CONDA_CUDATOOLKIT_CONSTRAINT="    - cudatoolkit >=9.0,<9.1 # [not osx]"
+    else
+	echo "unhandled desired_cuda: $desired_cuda"
+	exit 1
+    fi
+fi
+
+if [[ "$OSTYPE" == "msys" ]]; then
+    time conda build -c $ANACONDA_USER --no-anaconda-upload vs2017
+else
+    time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchvision
+fi
+time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchvision
+time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchvision
+time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchvision
+
+set +e

packaging/conda/switch_cuda_version.sh

+if [[ "$OSTYPE" == "msys" ]]; then
+    CUDA_DIR="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v$1"
+else
+    CUDA_DIR="/usr/local/cuda-$1"
+fi
+
+if ! ls "$CUDA_DIR"
+then
+    echo "folder $CUDA_DIR not found to switch"
+fi
+
+echo "Switching symlink to $CUDA_DIR"
+mkdir -p /usr/local
+rm -fr /usr/local/cuda
+ln -s "$CUDA_DIR" /usr/local/cuda
+
+if [[ "$OSTYPE" == "msys" ]]; then
+    export CUDA_VERSION=`ls /usr/local/cuda/bin/cudart64*.dll | head -1 | tr '._' ' ' | cut -d ' ' -f2`
+    export CUDNN_VERSION=`ls /usr/local/cuda/bin/cudnn64*.dll | head -1 | tr '._' ' ' | cut -d ' ' -f2`
+else
+    export CUDA_VERSION=$(ls /usr/local/cuda/lib64/libcudart.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev)
+    export CUDNN_VERSION=$(ls /usr/local/cuda/lib64/libcudnn.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev)
+fi
+
+ls -alh /usr/local/cuda
+
+echo "CUDA_VERSION=$CUDA_VERSION"
+echo "CUDNN_VERSION=$CUDNN_VERSION"

packaging/conda/torchvision/bld.bat

+@echo on
+
+set TORCHVISION_BUILD_VERSION=%PKG_VERSION%
+set TORCHVISION_BUILD_NUMBER=%PKG_BUILDNUM%
+
+if not "%CUDA_VERSION%" == "None" (
+    set build_with_cuda=1
+    set desired_cuda=%CUDA_VERSION:~0,-1%.%CUDA_VERSION:~-1,1%
+) else (
+    set build_with_cuda=
+)
+
+if "%build_with_cuda%" == "" goto cuda_flags_end
+
+set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda%
+set CUDA_BIN_PATH=%CUDA_PATH%\bin
+set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr
+if "%desired_cuda%" == "9.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
+if "%desired_cuda%" == "10.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50
+
+:cuda_flags_end
+
+python setup.py install --single-version-externally-managed --record=record.txt
+if errorlevel 1 exit /b 1

packaging/conda/torchvision/conda_build_config.yaml

+blas_impl:
+  - mkl                        # [x86_64]
+c_compiler:
+  - vs2017                     # [win]
+cxx_compiler:
+  - vs2017                     # [win]
+python:
+  - 3.5
+  - 3.6
+# This differs from target_platform in that it determines what subdir the compiler
+#    will target, not what subdir the compiler package will be itself.
+#    For example, we need a win-64 vs2008_win-32 package, so that we compile win-32
+#    code on win-64 miniconda.
+cross_compiler_target_platform:
+  - win-64                     # [win]
+target_platform:
+  - win-64                     # [win]
+vc:
+  - 14
+zip_keys:
+  -                             # [win]
+    - vc                        # [win]
+    - c_compiler                # [win]
+    - cxx_compiler              # [win]

packaging/conda/torchvision/meta.yaml

+package:
+  name: torchvision{{ environ.get('TORCHVISION_PACKAGE_SUFFIX') }}
+  version: "{{ environ.get('TORCHVISION_BUILD_VERSION') }}"
+
+source:
+ git_rev: v{{ environ.get('TORCHVISION_BUILD_VERSION') }}
+ git_url: https://github.com/pytorch/vision.git
+   
+
+requirements:
+  build:
+    - {{ compiler('c') }} # [win]
+
+  host:
+    - python
+    - setuptools
+    - pytorch{{ environ.get('TORCHVISION_PACKAGE_SUFFIX') }} >=1.1.0
+{{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }}
+
+  run:
+    - python
+    - pillow >=4.1.1
+    - numpy >=1.11
+    - pytorch{{ environ.get('TORCHVISION_PACKAGE_SUFFIX') }} >=1.1.0
+    - six
+{{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }}
+
+build:
+  number: {{ environ.get('TORCHVISION_BUILD_NUMBER') }}
+  string: py{{py}}_cu{{ environ['CUDA_VERSION'] }}_{{environ.get('TORCHVISION_BUILD_NUMBER')}}
+  script: python setup.py install --single-version-externally-managed --record=record.txt # [not win]
+  script_env:
+    - CUDA_VERSION
+
+test:
+  imports:
+    - torchvision
+    - torchvision.datasets
+    - torchvision.transforms
+  source_files:
+    - test
+  requires:
+    - pytest
+    - scipy
+  commands:
+    pytest .
+
+
+about:
+  home: https://github.com/pytorch/vision
+  license: BSD
+  license_file: LICENSE
+  summary: 'image and video datasets and models for torch deep learning'

packaging/wheel/linux_manywheel.sh

+if [ "$#" -ne 1 ]; then
+    echo "Illegal number of parameters. Pass cuda version"
+    echo "CUDA version should be cu90, cu100 or cpu"
+    exit 1
+fi
+export CUVER="$1" # cu90 cu100 cpu
+
+export TORCHVISION_BUILD_VERSION="0.3.0"
+export TORCHVISION_BUILD_NUMBER="1"
+export OUT_DIR="/remote/$CUVER"
+export TORCH_WHEEL="torch -f https://download.pytorch.org/whl/$CUVER/stable.html --no-index"
+
+pushd /opt/python
+DESIRED_PYTHON=(*/)
+popd
+for desired_py in "${DESIRED_PYTHON[@]}"; do
+    python_installations+=("/opt/python/$desired_py")
+done
+
+OLD_PATH=$PATH
+git clone https://github.com/pytorch/vision -b v${TORCHVISION_BUILD_VERSION}
+pushd vision
+for PYDIR in "${python_installations[@]}"; do
+    export PATH=$PYDIR/bin:$OLD_PATH
+    pip install numpy pyyaml future
+    pip install $TORCH_WHEEL
+    pip install ninja
+    python setup.py clean
+    python setup.py bdist_wheel
+    mkdir -p $OUT_DIR
+    cp dist/*.whl $OUT_DIR/
+done

packaging/wheel/osx_wheel.sh

+if [[ ":$PATH:" == *"conda"* ]]; then
+    echo "existing anaconda install in PATH, remove it and run script"
+    exit 1
+fi
+# download and activate anaconda
+rm -rf ~/minconda_wheel_env_tmp
+wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \
+    chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \
+    ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \
+    rm Miniconda3-latest-MacOSX-x86_64.sh
+
+. ~/minconda_wheel_env_tmp/bin/activate
+
+
+export TORCHVISION_BUILD_VERSION="0.3.0"
+export TORCHVISION_BUILD_NUMBER="1"
+export OUT_DIR=~/torchvision_wheels
+
+export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++
+
+pushd /tmp
+rm -rf vision
+git clone https://github.com/pytorch/vision -b v${TORCHVISION_BUILD_VERSION}
+pushd vision
+
+desired_pythons=( "2.7" "3.5" "3.6" "3.7" )
+# for each python
+for desired_python in "${desired_pythons[@]}"
+do
+    # create and activate python env
+    env_name="env$desired_python"
+    conda create -yn $env_name python="$desired_python"
+    conda activate $env_name
+
+    # install torchvision dependencies
+    pip install torch ninja scipy pytest
+
+    python setup.py clean
+    python setup.py bdist_wheel
+    mkdir -p $OUT_DIR
+    cp dist/*.whl $OUT_DIR/
+done
+popd
+popd