Run unit test on CCI (#576)

* wip

* Add test

* add caching

* fixup! add caching

* fixup! fixup! add caching

.circleci/config.yml

@@ -233,6 +233,36 @@ jobs:
             docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
             docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
 
+  unittest_linux:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - restore_cache:
+
+          keys:
+            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum "./packaging/build_from_source.sh" }}-{{ checksum ".circleci/unittest/environment.yml" }}
+
+      - run:
+          name: Setup
+          command: .circleci/unittest/setup_env.sh
+      - save_cache:
+
+          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum "./packaging/build_from_source.sh" }}-{{ checksum ".circleci/unittest/environment.yml" }}
+
+          paths:
+            - conda
+            - env
+            - third_party
+      - run:
+          name: Installation
+          command: .circleci/unittest/install.sh
+      - run:
+          name: Run test
+          command: .circleci/unittest/run_test.sh
+
 workflows:
   build:
     jobs:
@@ -276,7 +306,17 @@ workflows:
       - binary_win_conda:
           name: torchaudio_win_py3.6
           python_version: "3.6"
-
+  unittest:
+    jobs:
+      - unittest_linux:
+          name: unittest_linux_py3.6
+          python_version: '3.6'
+      - unittest_linux:
+          name: unittest_linux_py3.7
+          python_version: '3.7'
+      - unittest_linux:
+          name: unittest_linux_py3.8
+          python_version: '3.8'
   nightly:
     jobs:
       - circleci_consistency

.circleci/config.yml.in

@@ -233,19 +233,51 @@ jobs:
             docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
             docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
 
+  unittest_linux:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - restore_cache:
+          {% raw %}
+          keys:
+            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum "./packaging/build_from_source.sh" }}-{{ checksum ".circleci/unittest/environment.yml" }}
+          {% endraw %}
+      - run:
+          name: Setup
+          command: .circleci/unittest/setup_env.sh
+      - save_cache:
+          {% raw %}
+          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum "./packaging/build_from_source.sh" }}-{{ checksum ".circleci/unittest/environment.yml" }}
+          {% endraw %}
+          paths:
+            - conda
+            - env
+            - third_party
+      - run:
+          name: Installation
+          command: .circleci/unittest/install.sh
+      - run:
+          name: Run test
+          command: .circleci/unittest/run_test.sh
+
 workflows:
   build:
     jobs:
       - circleci_consistency
-      {{ workflows() }}
+      {{ build_workflows() }}
       - binary_win_conda:
           name: torchaudio_win_py3.6
           python_version: "3.6"
-
+  unittest:
+    jobs:
+      {{ unittest_workflows() }}
   nightly:
     jobs:
       - circleci_consistency
-      {{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
+      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
   docker_build:
     triggers:
       - schedule:

.circleci/regenerate.py

@@ -19,17 +19,20 @@ import yaml
 import os.path
 
 
-def workflows(prefix='', upload=False, filter_branch=None, indentation=6):
+PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]
+
+
+def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
     w = []
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos"]:
-            for python_version in ["3.6", "3.7", "3.8"]:
-                w += workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload)
+            for python_version in PYTHON_VERSIONS:
+                w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload)
 
     return indent(indentation, w)
 
 
-def workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False):
+def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False):
 
     w = []
     base_workflow_name = "{prefix}binary_{os_type}_{btype}_py{python_version}".format(
@@ -107,6 +110,19 @@ def indent(indentation, data_list):
     return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines())
 
 
+def unittest_workflows(indentation=6):
+    w = []
+    for os_type in ["linux"]:
+        for python_version in PYTHON_VERSIONS:
+            w.append({
+                f"unittest_{os_type}": {
+                    "name": f"unittest_{os_type}_py{python_version}",
+                    "python_version": python_version,
+                }
+            })
+    return indent(indentation, w)
+
+
 if __name__ == "__main__":
     d = os.path.dirname(__file__)
     env = jinja2.Environment(
@@ -116,4 +132,7 @@ if __name__ == "__main__":
     )
 
     with open(os.path.join(d, 'config.yml'), 'w') as f:
-        f.write(env.get_template('config.yml.in').render(workflows=workflows))
+        f.write(env.get_template('config.yml.in').render(
+            build_workflows=build_workflows,
+            unittest_workflows=unittest_workflows,
+        ))

.circleci/unittest/environment.yml

+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - flake8
+  - numpy
+  - pytest
+  - librosa
+  - pip
+  - pip:
+    - kaldi-io
+    - scipy

.circleci/unittest/install.sh

+#!/usr/bin/env bash
+
+unset PYTORCH_VERSION
+# For unittest, nightly PyTorch is used, and we do not want to fixiate on the version
+
+set -e
+
+eval "$(./conda/bin/conda shell.bash hook)"
+conda activate ./env
+
+printf "* Installing PyTorch nightly build"
+conda install -c pytorch-nightly pytorch cpuonly
+
+printf "* Setting up torchaudio\n"
+IS_CONDA=true python setup.py develop

.circleci/unittest/run_test.sh

+#!/usr/bin/env bash
+
+set -e
+
+eval "$(./conda/bin/conda shell.bash hook)"
+conda activate ./env
+
+python -m torch.utils.collect_env
+pytest -v test

.circleci/unittest/setup_env.sh

+#!/usr/bin/env bash
+
+# This script is for setting up environment for running unit test on CircleCI.
+# To speed up the CI time, the result of environment is cached.
+# PyTorch is not included here, so that it won't be cached.
+
+set -e
+
+this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+root_dir="$(git rev-parse --show-toplevel)"
+conda_dir="${root_dir}/conda"
+env_dir="${root_dir}/env"
+
+cd "${root_dir}"
+
+# 1. Install conda at ./conda
+if [ ! -d "${conda_dir}" ]; then
+    printf "* Installing conda\n"
+    wget -O miniconda.sh http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    bash ./miniconda.sh -b -f -p "${conda_dir}"
+fi
+printf "* Checking conda update\n"
+eval "$(${conda_dir}/bin/conda shell.bash hook)"
+conda update -n base -c defaults conda
+
+# 2. Create test environment at ./env
+if [ ! -d "${env_dir}" ]; then
+    printf "* Creating a test environment\n"
+    conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
+fi
+printf "* Installing dependencies (except PyTorch)\n"
+conda activate "${env_dir}"
+conda env update --file "${this_dir}/environment.yml" --prune
+
+# 3. Build codecs at ./third_party
+if [ ! -d "./third_party" ]; then
+    printf "* Building Codecs"
+    ./packaging/build_from_source.sh "$PWD"
+fi

test/test_librosa_compatibility.py

@@ -243,10 +243,8 @@ class TestTransforms(_LibrosaMixin, unittest.TestCase):
         }
         _test_compatibilities(**kwargs)
 
-    # NOTE: Test passes offline, but fails on TravisCI, see #372.
-    @unittest.skipIf(
-        os.environ.get('CI') == 'true' and os.environ.get('TRAVIS') == 'true',
-        'Test is known to fail on TravisCI')
+    # NOTE: Test passes offline, but fails on TravisCI (and CircleCI), see #372.
+    @unittest.skipIf('CI' in os.environ, 'Test is known to fail on CI')
     def test_basics3(self):
         kwargs = {
             'n_fft': 200,