Enable unittest jobs for windows (#604)

* Turn on tests when building through conda-build

* skip one failling test on windows.

* Switch to unittest

* Fix conda env activation

* add linux only condition

* Fix librosa installation

* conda-forge channel as a fallback

* Revert "conda-forge channel as a fallback"

This reverts commit 8c15f6638e34792640a3de3e057d9c2995425180.

* Try the pytorch way

* Use conda-forge

* Disable conda-forge

* Install future at setup env

* Move linux unittest CI job tools to dedicated directory

* Restore Linux scripts and move the updated souces to a new subdir

* Add back future

* Remove comments
Co-authored-by: Vincent Quenneville-Belair <vincentqb@gmail.com>
Co-authored-by: moto <855818+mthrok@users.noreply.github.com>

.circleci/config.yml

@@ -6,8 +6,12 @@ version: 2.1
 #     - Replace binary_linux_wheel_py3.8 with the name of the job you want to test.
 #       Job names are 'name:' key.
 
-orbs:
-  win: circleci/windows@2.0.0
+executors:
+  windows-cpu:
+    machine:
+      resource_class: windows.xlarge
+      image: windows-server-2019-vs2019:stable
+      shell: bash.exe
 
 binary_common: &binary_common
   parameters:
@@ -80,22 +84,6 @@ jobs:
           paths:
             - "*"
 
-  binary_win_conda:
-    <<: *binary_common
-    executor:
-      name: win/default
-      shell: bash.exe
-    steps:
-      - checkout
-      - run:
-          command: |
-            choco install miniconda3
-            (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
-            conda activate base
-            conda install -yq conda-build "conda-package-handling!=1.5.0"
-            bash packaging/build_conda.sh
-          shell: powershell.exe
-
   binary_macos_wheel:
     <<: *binary_common
     macos:
@@ -247,27 +235,64 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+
+      - run:
+          name: Setup
+          command: .circleci/unittest/linux/scripts/setup_env.sh
+      - save_cache:
+
+          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+
+          paths:
+            - conda
+            - env
+      - run:
+          name: Install torchaudio
+          command: .circleci/unittest/linux/scripts/install.sh
+      - run:
+          name: Run tests
+          command: .circleci/unittest/linux/scripts/run_test.sh
+      - run:
+          name: Post process
+          command: .circleci/unittest/linux/scripts/post_process.sh
+      - store_test_results:
+          path: test-results
+
+  unittest_windows:
+    <<: *binary_common
+    executor:
+      name: windows-cpu
+    steps:
+      - checkout
+      - run:
+          name: Generate cache key
+          # This will refresh cache on Sundays, nightly build should generate new cache.
+          command: echo "$(date +"%Y-%U")" > .circleci-weekly
+      - restore_cache:
+
+          keys:
+            - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
 
       - run:
           name: Setup
-          command: .circleci/unittest/scripts/setup_env.sh
+          command: .circleci/unittest/windows/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
 
           paths:
             - conda
             - env
       - run:
           name: Install torchaudio
-          command: .circleci/unittest/scripts/install.sh
+          command: .circleci/unittest/windows/scripts/install.sh
       - run:
           name: Run tests
-          command: .circleci/unittest/scripts/run_test.sh
+          command: .circleci/unittest/windows/scripts/run_test.sh
       - run:
           name: Post process
-          command: .circleci/unittest/scripts/post_process.sh
+          command: .circleci/unittest/windows/scripts/post_process.sh
       - store_test_results:
           path: test-results
 
@@ -311,9 +336,6 @@ workflows:
       - binary_macos_conda:
           name: binary_macos_conda_py3.8
           python_version: '3.8'
-      - binary_win_conda:
-          name: torchaudio_win_py3.6
-          python_version: "3.6"
   unittest:
     jobs:
       - unittest_linux:
@@ -325,6 +347,15 @@ workflows:
       - unittest_linux:
           name: unittest_linux_py3.8
           python_version: '3.8'
+      - unittest_windows:
+          name: unittest_windows_py3.6
+          python_version: '3.6'
+      - unittest_windows:
+          name: unittest_windows_py3.7
+          python_version: '3.7'
+      - unittest_windows:
+          name: unittest_windows_py3.8
+          python_version: '3.8'
   nightly:
     jobs:
       - circleci_consistency:

.circleci/config.yml.in

@@ -6,8 +6,12 @@ version: 2.1
 #     - Replace binary_linux_wheel_py3.8 with the name of the job you want to test.
 #       Job names are 'name:' key.
 
-orbs:
-  win: circleci/windows@2.0.0
+executors:
+  windows-cpu:
+    machine:
+      resource_class: windows.xlarge
+      image: windows-server-2019-vs2019:stable
+      shell: bash.exe
 
 binary_common: &binary_common
   parameters:
@@ -80,22 +84,6 @@ jobs:
           paths:
             - "*"
 
-  binary_win_conda:
-    <<: *binary_common
-    executor:
-      name: win/default
-      shell: bash.exe
-    steps:
-      - checkout
-      - run:
-          command: |
-            choco install miniconda3
-            (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
-            conda activate base
-            conda install -yq conda-build "conda-package-handling!=1.5.0"
-            bash packaging/build_conda.sh
-          shell: powershell.exe
-
   binary_macos_wheel:
     <<: *binary_common
     macos:
@@ -247,27 +235,64 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+          {% endraw %}
+      - run:
+          name: Setup
+          command: .circleci/unittest/linux/scripts/setup_env.sh
+      - save_cache:
+          {% raw %}
+          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+          {% endraw %}
+          paths:
+            - conda
+            - env
+      - run:
+          name: Install torchaudio
+          command: .circleci/unittest/linux/scripts/install.sh
+      - run:
+          name: Run tests
+          command: .circleci/unittest/linux/scripts/run_test.sh
+      - run:
+          name: Post process
+          command: .circleci/unittest/linux/scripts/post_process.sh
+      - store_test_results:
+          path: test-results
+
+  unittest_windows:
+    <<: *binary_common
+    executor:
+      name: windows-cpu
+    steps:
+      - checkout
+      - run:
+          name: Generate cache key
+          # This will refresh cache on Sundays, nightly build should generate new cache.
+          command: echo "$(date +"%Y-%U")" > .circleci-weekly
+      - restore_cache:
+          {% raw %}
+          keys:
+            - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
           {% endraw %}
       - run:
           name: Setup
-          command: .circleci/unittest/scripts/setup_env.sh
+          command: .circleci/unittest/windows/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
           {% endraw %}
           paths:
             - conda
             - env
       - run:
           name: Install torchaudio
-          command: .circleci/unittest/scripts/install.sh
+          command: .circleci/unittest/windows/scripts/install.sh
       - run:
           name: Run tests
-          command: .circleci/unittest/scripts/run_test.sh
+          command: .circleci/unittest/windows/scripts/run_test.sh
       - run:
           name: Post process
-          command: .circleci/unittest/scripts/post_process.sh
+          command: .circleci/unittest/windows/scripts/post_process.sh
       - store_test_results:
           path: test-results
 
@@ -276,9 +301,6 @@ workflows:
     jobs:
       - circleci_consistency
       {{ build_workflows() }}
-      - binary_win_conda:
-          name: torchaudio_win_py3.6
-          python_version: "3.6"
   unittest:
     jobs:
       {{ unittest_workflows() }}

.circleci/regenerate.py

@@ -112,7 +112,7 @@ def indent(indentation, data_list):
 
 def unittest_workflows(indentation=6):
     w = []
-    for os_type in ["linux"]:
+    for os_type in ["linux", "windows"]:
         for python_version in PYTHON_VERSIONS:
             w.append({
                 f"unittest_{os_type}": {

.circleci/unittest/windows/README.md

+This directory contains;
+
+ - scripts
+   Scripts used by CircleCI to run unit tests.

.circleci/unittest/windows/scripts/environment.yml

+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - flake8
+  - numpy
+  - pytest
+  - pytest-cov
+  - codecov
+  - pip
+  - pip:
+    - scipy
+    - kaldi-io
+    - PySoundFile
+    - librosa
+    - future

.circleci/unittest/windows/scripts/install.sh

+#!/usr/bin/env bash
+
+unset PYTORCH_VERSION
+# For unittest, nightly PyTorch is used as the following section,
+# so no need to set PYTORCH_VERSION.
+# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
+
+set -e
+
+eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')"
+conda activate ./env
+
+printf "* Installing PyTorch nightly build"
+conda install -y -c pytorch-nightly pytorch cpuonly
+
+printf "* Installing torchaudio\n"
+IS_CONDA=true python setup.py develop

.circleci/unittest/windows/scripts/install_conda.bat

+start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%

.circleci/unittest/windows/scripts/post_process.sh

+#!/usr/bin/env bash
+
+set -e
+
+eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')"
+conda activate ./env
+
+codecov

.circleci/unittest/windows/scripts/run_test.sh

+#!/usr/bin/env bash
+
+set -e
+
+eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')"
+conda activate ./env
+
+python -m torch.utils.collect_env
+pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test
+flake8 torchaudio test