Do not auto-skip tests on CI (#2127)

Summary:
Update the internal of `skipIfXXX` decorators so that tests in CI will not be automatically skipped.

Currently we automatically skip some tests based on the availability of related features/test tools.
This causes issues where we miss signals on certain important features. (CUDA on Windows) https://github.com/pytorch/audio/issues/1565

The new `skipIf` decorator will fail if in CI unless it is explicitly allowed to skip tests.
It does so by checking `CI` and `TORCHAUDIO_TEST_ALLOW_SKIP_IF_XXX` environment variables.

For non-CI environments, the behavior is same as before, but users can now set `TORCHAUDIO_TEST_ALLOW_SKIP_IF_XXX=false` to disallow the automatic skip.

Results without `TORCHAUDIO_TEST_ALLOW_SKIP_IF_XXX` https://app.circleci.com/pipelines/github/pytorch/audio/9112/workflows/4e6db046-a1a2-4965-b0fe-d5baf4a1efac

Pull Request resolved: https://github.com/pytorch/audio/pull/2127

Reviewed By: hwangjeff

Differential Revision: D33430711

Pulled By: mthrok

fbshipit-source-id: d8954dd720469c5ab0f34ea062fd8cf04a8afa3e

.circleci/config.yml

@@ -460,6 +460,8 @@ jobs:
       - run:
           name: Run tests
           command: .circleci/unittest/linux/scripts/run_test.sh
+          environment:
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -489,7 +491,7 @@ jobs:
           command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
-          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "TORCHAUDIO_TEST_FORCE_CUDA=1" -e "CI=${CI}" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
+          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "CI=${CI}" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -512,6 +514,17 @@ jobs:
       - run:
           name: Run tests
           command: .circleci/unittest/windows/scripts/run_test.sh
+          environment:
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX: true
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -524,7 +537,6 @@ jobs:
     environment:
       <<: *environment
       CUDA_VERSION: "11.3"
-      TORCHAUDIO_TEST_FORCE_CUDA: 1
     steps:
       - checkout
       - designate_upload_channel
@@ -544,6 +556,16 @@ jobs:
       - run:
           name: Run tests
           command: .circleci/unittest/windows/scripts/run_test.sh
+          environment:
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX: true
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -569,6 +591,14 @@ jobs:
       - run:
           name: Run tests
           command: .circleci/unittest/linux/scripts/run_test.sh
+          environment:
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_QUANTIZATION: true
       - store_test_results:
           path: test-results
       - store_artifacts:

.circleci/config.yml.in

@@ -460,6 +460,8 @@ jobs:
       - run:
           name: Run tests
           command: .circleci/unittest/linux/scripts/run_test.sh
+          environment:
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -489,7 +491,7 @@ jobs:
           command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
-          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "TORCHAUDIO_TEST_FORCE_CUDA=1" -e "CI=${CI}" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
+          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "CI=${CI}" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -512,6 +514,17 @@ jobs:
       - run:
           name: Run tests
           command: .circleci/unittest/windows/scripts/run_test.sh
+          environment:
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX: true
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -524,7 +537,6 @@ jobs:
     environment:
       <<: *environment
       CUDA_VERSION: "11.3"
-      TORCHAUDIO_TEST_FORCE_CUDA: 1
     steps:
       - checkout
       - designate_upload_channel
@@ -544,6 +556,16 @@ jobs:
       - run:
           name: Run tests
           command: .circleci/unittest/windows/scripts/run_test.sh
+          environment:
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX: true
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -569,6 +591,14 @@ jobs:
       - run:
           name: Run tests
           command: .circleci/unittest/linux/scripts/run_test.sh
+          environment:
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
+              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_QUANTIZATION: true
       - store_test_results:
           path: test-results
       - store_artifacts:

.circleci/unittest/linux/scripts/run_test.sh

@@ -5,14 +5,9 @@ set -e
 eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
-case "$(uname -s)" in
-    Darwin*) os=MacOSX;;
-    *) os=Linux
-esac
-
 python -m torch.utils.collect_env
+env | grep TORCHAUDIO || true
 
-export TORCHAUDIO_TEST_FAIL_IF_NO_EXTENSION=1
 export PATH="${PWD}/third_party/install/bin/:${PATH}"
 
 declare -a args=(

.circleci/unittest/windows/scripts/run_test.sh

@@ -9,6 +9,8 @@ this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$this_dir/set_cuda_envs.sh"
 
 python -m torch.utils.collect_env
+env | grep TORCHAUDIO || true
+
 cd test
 pytest --cov=torchaudio --junitxml=../test-results/junit.xml -v --durations 20 torchaudio_unittest
 coverage html

test/torchaudio_unittest/common_utils/case_utils.py

+import functools
 import os.path
 import shutil
 import subprocess
@@ -94,32 +95,107 @@ class TorchaudioTestCase(TestBaseMixin, PytorchTestCase):
     pass
 
 
+def _eval_env(var, default):
+    if var not in os.environ:
+        return default
+
+    val = os.environ.get(var, "0")
+    trues = ["1", "true", "TRUE", "on", "ON", "yes", "YES"]
+    falses = ["0", "false", "FALSE", "off", "OFF", "no", "NO"]
+    if val in trues:
+        return True
+    if val not in falses:
+        # fmt: off
+        raise RuntimeError(
+            f"Unexpected environment variable value `{var}={val}`. "
+            f"Expected one of {trues + falses}")
+        # fmt: on
+    return False
+
+
+def _fail(reason):
+    def deco(test_item):
+        if isinstance(test_item, type):
+            # whole class is decorated
+            def _f(self, *_args, **_kwargs):
+                raise RuntimeError(reason)
+
+            test_item.setUp = _f
+            return test_item
+
+        # A method is decorated
+        @functools.wraps(test_item)
+        def f(*_args, **_kwargs):
+            raise RuntimeError(reason)
+
+        return f
+
+    return deco
+
+
+def _pass(test_item):
+    return test_item
+
+
+_IN_CI = _eval_env("CI", default=False)
+
+
+def _skipIf(condition, reason, key):
+    if not condition:
+        return _pass
+
+    # In CI, default to fail, so as to prevent accidental skip.
+    # In other env, default to skip
+    var = f"TORCHAUDIO_TEST_ALLOW_SKIP_IF_{key}"
+    skip_allowed = _eval_env(var, default=not _IN_CI)
+    if skip_allowed:
+        return unittest.skip(reason)
+    return _fail(f"{reason} But the test cannot be skipped. (CI={_IN_CI}, {var}={skip_allowed}.)")
+
+
 def skipIfNoExec(cmd):
-    return unittest.skipIf(shutil.which(cmd) is None, f"`{cmd}` is not available")
+    return _skipIf(
+        shutil.which(cmd) is None,
+        f"`{cmd}` is not available.",
+        key=f"NO_CMD_{cmd.upper().replace('-', '_')}",
+    )
 
 
 def skipIfNoModule(module, display_name=None):
-    display_name = display_name or module
-    return unittest.skipIf(not is_module_available(module), f'"{display_name}" is not available')
-
-
-def skipIfNoCuda(test_item):
-    if torch.cuda.is_available():
-        return test_item
-    force_cuda_test = os.environ.get("TORCHAUDIO_TEST_FORCE_CUDA", "0")
-    if force_cuda_test not in ["0", "1"]:
-        raise ValueError('"TORCHAUDIO_TEST_FORCE_CUDA" must be either "0" or "1".')
-    if force_cuda_test == "1":
-        raise RuntimeError('"TORCHAUDIO_TEST_FORCE_CUDA" is set but CUDA is not available.')
-    return unittest.skip("CUDA is not available.")(test_item)
-
-
-skipIfNoSox = unittest.skipIf(not is_sox_available(), reason="Sox not available")
-skipIfNoKaldi = unittest.skipIf(not is_kaldi_available(), reason="Kaldi not available")
-skipIfNoCtcDecoder = unittest.skipIf(not is_ctc_decoder_available(), reason="CTC decoder not available")
-skipIfRocm = unittest.skipIf(
-    os.getenv("TORCHAUDIO_TEST_WITH_ROCM", "0") == "1", reason="test doesn't currently work on the ROCm stack"
+    return _skipIf(
+        not is_module_available(module),
+        f'"{display_name or module}" is not available.',
+        key=f"NO_MOD_{module.replace('.', '_')}",
+    )
+
+
+skipIfNoCuda = _skipIf(
+    not torch.cuda.is_available(),
+    reason="CUDA is not available.",
+    key="NO_CUDA",
+)
+skipIfNoSox = _skipIf(
+    not is_sox_available(),
+    reason="Sox features are not available.",
+    key="NO_SOX",
+)
+skipIfNoKaldi = _skipIf(
+    not is_kaldi_available(),
+    reason="Kaldi features are not available.",
+    key="NO_KALDI",
+)
+skipIfNoCtcDecoder = _skipIf(
+    not is_ctc_decoder_available(),
+    reason="CTC decoder not available.",
+    key="NO_CTC_DECODER",
+)
+skipIfRocm = _skipIf(
+    _eval_env("TORCHAUDIO_TEST_WITH_ROCM", default=False),
+    reason="The test doesn't currently work on the ROCm stack.",
+    key="ON_ROCM",
 )
-skipIfNoQengine = unittest.skipIf(
-    "fbgemm" not in torch.backends.quantized.supported_engines, reason="`fbgemm` is not available."
+skipIfNoQengine = _skipIf(
+    "fbgemm" not in torch.backends.quantized.supported_engines,
+    reason="`fbgemm` is not available.",
+    key="NO_QUANTIZATION",
 )