Add CircleCI pipeline for Linux binary builds. (#214)

Signed-off-by: Edward Z. Yang <ezyang@fb.com>

.circleci/config.yml

+version: 2.1
+
+# How to test:
+#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
+#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel
+#     - Replace binary_linux_wheel with the name of the job you want to test
+
+param_target_commit: &param_target_commit
+  target_commit:
+    description: "what hash or branch to build binaries for; by default, same as trigger"
+    type: string
+    default: HEAD
+
+jobs:
+  binary_linux_wheel:
+    parameters:
+      <<: *param_target_commit
+    docker:
+      - image: "soumith/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/wheel/linux_manywheel.sh
+      - store_artifacts:
+          path: /remote/cpu
+
+  binary_linux_conda:
+    parameters:
+      <<: *param_target_commit
+    docker:
+      - image: "soumith/conda-cuda"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: TARGET_COMMIT=<< parameters.target_commit >> build_tools/packaging/conda/build_audio.sh
+      - store_artifacts:
+          path: /opt/conda/conda-bld/linux-64
+
+workflows:
+  build:
+    jobs:
+      - binary_linux_wheel
+      - binary_linux_conda

.gitignore

@@ -109,3 +109,4 @@ ENV/
 # Generated Files
 test/assets/sinewave.wav
 torchaudio/version.py
+gen.yml

build_tools/packaging/README.md

@@ -9,7 +9,6 @@
 nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash
 pushd remote/conda
 conda config --add channels pytorch
-conda config --add channels conda-forge
 ./build_audio.sh
 ```
 

build_tools/packaging/conda/build_audio.sh

@@ -10,31 +10,38 @@ retry () {
     $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
 }
 
-export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")"
-export TORCHAUDIO_BUILD_NUMBER=1
-
-SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-audio_rootdir="$(pwd)/torchaudio-src"
-
-if [[ "$BRANCH" == "" ]]; then
-  BRANCH=master
+if [[ -z "$TORCHAUDIO_BUILD_VERSION" ]]; then
+  export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")"
+fi
+if [[ -z "$TORCHAUDIO_BUILD_NUMBER" ]]; then
+  export TORCHAUDIO_BUILD_NUMBER=1
 fi
 
-if [[ ! -d "$audio_rootdir" ]]; then
-    rm -rf "$audio_rootdir"
-    git clone "https://github.com/pytorch/audio" "$audio_rootdir" -b "$BRANCH"
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+if [[ "$TARGET_COMMIT" == HEAD ]]; then
+  # Assume that this script was called from a valid checkout
+  WORKDIR="$(realpath "$script_dir/../../..")"
+else
+  WORKDIR="/tmp/audio"
+  cd /tmp
+  rm -rf audio
+  git clone https://github.com/pytorch/audio
+  cd audio
+  git checkout "$TARGET_COMMIT"
+  git submodule update --init --recursive
+  mkdir audio/third_party
 fi
 
-export TORCHAUDIO_GITHUB_ROOT_DIR="$audio_rootdir"
+export TORCHAUDIO_GITHUB_ROOT_DIR="$WORKDIR"
 
-cd "$SOURCE_DIR"
+cd "$script_dir"
 
-ANACONDA_USER=pytorch
+ANACONDA_USER=pytorch-nightly
 conda config --set anaconda_upload no
 
-LATEST_PYTORCH_NIGHTLY_VERSION=$(conda search --json 'pytorch-nightly[channel=pytorch]' | python "$SOURCE_DIR/get-latest.py")
-export CONDA_PYTORCH_CONSTRAINT="    - pytorch-nightly ==${LATEST_PYTORCH_NIGHTLY_VERSION}"
+LATEST_PYTORCH_NIGHTLY_VERSION=$(conda search --json 'pytorch[channel=pytorch-nightly]' | python "$script_dir/get-latest.py")
+export CONDA_PYTORCH_CONSTRAINT="    - pytorch ==${LATEST_PYTORCH_NIGHTLY_VERSION}"
 export CONDA_CUDATOOLKIT_CONSTRAINT=""
 export CUDA_VERSION="None"
 if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -42,9 +49,6 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
 fi
 
 time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchaudio
-# Currently disabled, see https://github.com/pytorch/pytorch/issues/23738
-# time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio
+time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio
 time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchaudio
 time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchaudio
-
- set +e

build_tools/packaging/conda/get-latest.py

 # Functionally equivalent to:
-# jq -r '.["pytorch-nightly"][-1].version' | sed 's/+.*$//'
+# jq -r '.["pytorch"][-1].version' | sed 's/+.*$//'
 
 import sys
 import json
 import re
 
-print(re.sub(r'\+.*$', '', json.load(sys.stdin)["pytorch-nightly"][-1]["version"]))
+print(re.sub(r'\+.*$', '', json.load(sys.stdin)["pytorch"][-1]["version"]))

build_tools/packaging/wheel/linux_manywheel.sh

@@ -2,44 +2,58 @@
 
 set -ex
 
-export TORCHAUDIO_PACKAGE_NAME="torchaudio_nightly"
-export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")"
-export TORCHAUDIO_BUILD_NUMBER="1"
+if [[ -z "$TORCHAUDIO_BUILD_VERSION" ]]; then
+  export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")"
+fi
+if [[ -z "$TORCHAUDIO_BUILD_NUMBER" ]]; then
+  export TORCHAUDIO_BUILD_NUMBER="1"
+fi
 export OUT_DIR="/remote/cpu"
 
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
 cd /opt/python
 DESIRED_PYTHON=(*/)
 for desired_py in "${DESIRED_PYTHON[@]}"; do
     python_installations+=("/opt/python/$desired_py")
 done
 
-OLD_PATH=$PATH
-cd /tmp
-rm -rf audio
-git clone https://github.com/pytorch/audio
-mkdir audio/third_party
+if [[ "$TARGET_COMMIT" == HEAD ]]; then
+  # Assume that this script was called from a valid checkout
+  WORKDIR="$(realpath "$script_dir/../../..")"
+else
+  WORKDIR="/tmp/audio"
+  cd /tmp
+  rm -rf audio
+  git clone https://github.com/pytorch/audio
+  cd audio
+  git checkout "$TARGET_COMMIT"
+  git submodule update --init --recursive
+fi
+
+mkdir "$WORKDIR/third_party"
 
-export PREFIX="/tmp/audio"
-. /remote/wheel/build_from_source.sh
+export PREFIX="$WORKDIR"
+. "$script_dir/build_from_source.sh"
 
-cd /tmp/audio
+cd "$WORKDIR"
 
+ORIG_TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION"
+
+OLD_PATH=$PATH
 for PYDIR in "${python_installations[@]}"; do
-    export PATH=$PYDIR/bin:$OLD_PATH
+    export PATH="$PYDIR/bin:$OLD_PATH"
     pip install --upgrade pip
 
-    # For true hermetic builds, you ought to be constructing the docker
-    # from scratch each time.  But this makes things marginally safer if
-    # you aren't doing this.
-    pip uninstall -y torch || true
-    pip uninstall -y torch_nightly || true
-
-    export TORCHAUDIO_PYTORCH_DEPENDENCY_NAME=torch_nightly
-    pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
-    # CPU/CUDA variants of PyTorch have ABI compatible PyTorch.  Therefore, we
-    # strip off the local package qualifier.  Also, we choose to build against
-    # the CPU build, because it takes less time to download.
-    export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')"
+    if [[ -z "$ORIG_TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" ]]; then
+      pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+      # CPU/CUDA variants of PyTorch have ABI compatible PyTorch.  Therefore, we
+      # strip off the local package qualifier.  Also, we choose to build against
+      # the CPU build, because it takes less time to download.
+      export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')"
+    else
+      pip install "torch==$TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION" -f https://download.pytorch.org/whl/torch_stable.html
+    fi
     echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}"
 
     # NB: do not actually install requirements.txt; that is only needed for