Clean up CI scripts (#3407)

Summary:
- Moving the unit test scripts from .circleci to .github
- Remove docker file for unit test base
- Use the Conda from Docker image in Linux jobs.

Remaining follow-up items

- Reuse the unittest script in Linux GPU job like done in Linux CPU job.

The unit test script needs to be fixed to be used for Linux GPU job
in new GHA workflow. Keeping it as a separate follow-up work item.

Pull Request resolved: https://github.com/pytorch/audio/pull/3407

Differential Revision: D46498263

Pulled By: mthrok

fbshipit-source-id: d8256717a55bb4257151d819d3b2ebd453601eac

.circleci/unittest/linux/README.md

-This directory contains;
-
- - docker
-   Docker image definition and scripts to build and update Docker image for unittest.
- - scripts
-   Scripts used by CircleCI to run unit tests.

.circleci/unittest/linux/docker/.dockerignore

-*
-!scripts

.circleci/unittest/linux/docker/.gitignore

-scripts/build_third_parties.sh
-Dockerfile.tmp

.circleci/unittest/linux/docker/Dockerfile

-FROM ubuntu:18.04 as builder
-
-RUN apt update -q
-
-################################################################################
-# Build Kaldi
-################################################################################
-RUN apt install -q -y \
-        autoconf \
-        automake \
-        bzip2 \
-        g++ \
-        gfortran \
-        git \
-        libatlas-base-dev \
-        libtool \
-        make \
-        python2.7 \
-        python3 \
-        sox \
-        subversion \
-        unzip \
-        wget \
-        zlib1g-dev
-
-# KALDI uses MKL as a default math library, but we are going to copy featbin binaries and dependent
-# shared libraries to the final image, so we use ATLAS, which is easy to reinstall in the final image.
-RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \
-        cd /opt/kaldi/tools && \
-        make -j $(nproc) && \
-        cd /opt/kaldi/src && \
-        ./configure --shared --mathlib=ATLAS --use-cuda=no && \
-        make featbin -j $(nproc)
-
-# Copy featbins and dependent libraries
-ADD ./scripts /scripts
-RUN bash /scripts/copy_kaldi_executables.sh /opt/kaldi /kaldi
-
-################################################################################
-# Build the final image
-################################################################################
-FROM BASE_IMAGE
-RUN apt update && apt install -y \
-        g++ \
-        gfortran \
-        git \
-        libatlas3-base \
-        libsndfile1 \
-        wget \
-        curl \
-        make \
-        file \
-        pkg-config \
-    && rm -rf /var/lib/apt/lists/*
-COPY --from=builder /kaldi /kaldi
-ENV PATH="${PATH}:/kaldi/bin" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/kaldi/lib"

.circleci/unittest/linux/docker/build_and_push.sh

-#!/usr/bin/env bash
-
-set -euo pipefail
-
-if [ $# -ne 1 ]; then
-    printf "Usage %s <CUDA_VERSION>\n\n" "$0"
-    exit 1
-fi
-
-datestr="$(date "+%Y%m%d")"
-if [ "$1" = "cpu" ]; then
-    base_image="ubuntu:18.04"
-    image="pytorch/torchaudio_unittest_base:manylinux-${datestr}"
-else
-    base_image="nvidia/cuda:$1-devel-ubuntu18.04"
-    docker pull "${base_image}"
-    image="pytorch/torchaudio_unittest_base:manylinux-cuda$1-${datestr}"
-fi
-
-cd "$( dirname "${BASH_SOURCE[0]}" )"
-
-# docker build also accepts reading from STDIN
-# but in that case, no context (other files) can be passed, so we write out Dockerfile
-sed "s|BASE_IMAGE|${base_image}|g" Dockerfile > Dockerfile.tmp
-docker build -t "${image}" -f Dockerfile.tmp .
-docker push "${image}"

.circleci/unittest/linux/docker/scripts/copy_kaldi_executables.sh

-#!/usr/bin/env bash
-
-list_executables() {
-    # List up executables in the given directory
-    find "$1" -type f -executable
-}
-
-list_kaldi_libraries() {
-    # List up shared libraries used by executables found in the given directory ($1)
-    # that reside in Kaldi directory ($2)
-    while read file; do
-        ldd "${file}" | grep -o "${2}.* ";
-    done < <(list_executables "$1") | sort -u
-}
-
-set -euo pipefail
-
-kaldi_root="$(realpath "$1")"
-target_dir="$(realpath "$2")"
-
-bin_dir="${target_dir}/bin"
-lib_dir="${target_dir}/lib"
-
-mkdir -p "${bin_dir}" "${lib_dir}"
-
-# 1. Copy featbins
-printf "Copying executables to %s\n" "${bin_dir}"
-while read file; do
-    printf "  %s\n" "${file}"
-    cp "${file}" "${bin_dir}"
-done < <(list_executables "${kaldi_root}/src/featbin")
-
-# 2. Copy dependent libraries from Kaldi
-printf "Copying libraries to %s\n" "${lib_dir}"
-while read file; do
-    printf "  %s\n" "$file"
-    # If it is not symlink, just copy to the target directory
-    if [ ! -L "${file}" ]; then
-        cp "${file}" "${lib_dir}"
-        continue
-    fi
-
-    # If it is symlink,
-    # 1. Copy the actual library to the target directory.
-    library="$(realpath "${file}")"
-    cp "${library}" "${lib_dir}"
-    # 2. then if the name of the symlink is different from the actual library name,
-    #    create the symlink in the target directory.
-    lib_name="$(basename "${library}")"
-    link_name="$(basename "${file}")"
-    if [ "${lib_name}" != "${link_name}" ]; then
-        printf "    Linking %s -> %s\n" "${lib_name}" "${link_name}"
-        (
-            cd "${lib_dir}"
-            ln -sf "${lib_name}" "${link_name}"
-        )
-    fi
-done < <(list_kaldi_libraries "${bin_dir}" "${kaldi_root}")

.circleci/unittest/linux/scripts/setup_env.sh

-#!/usr/bin/env bash
-
-# This script is for setting up environment in which unit test is ran.
-# To speed up the CI time, the resulting environment is cached.
-#
-# Do not install PyTorch and torchaudio here, otherwise they also get cached.
-
-set -ex
-
-root_dir="$(git rev-parse --show-toplevel)"
-conda_dir="${root_dir}/conda"
-env_dir="${root_dir}/env"
-
-cd "${root_dir}"
-
-case "$(uname -s)" in
-    Darwin*) os=MacOSX;;
-    *) os=Linux
-esac
-
-# 1. Install conda at ./conda
-if [ ! -d "${conda_dir}" ]; then
-    printf "* Installing conda\n"
-    curl --silent -L -o miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh"
-    bash ./miniconda.sh -b -f -p "${conda_dir}"
-fi
-eval "$("${conda_dir}/bin/conda" shell.bash hook)"
-
-
-# 2. Create test environment at ./env
-if [ ! -d "${env_dir}" ]; then
-    printf "* Creating a test environment with PYTHON_VERSION=%s\n" "${PYTHON_VERSION}\n"
-    conda create --prefix "${env_dir}" -y python="${PYTHON_VERSION}"
-fi
-conda activate "${env_dir}"
-
-# 3. Install minimal build tools
-pip --quiet install cmake ninja
-conda install --quiet -y 'ffmpeg>=4.1' pkg-config

.circleci/unittest/windows/README.md

-This directory contains;
-
- - scripts
-   Scripts used by CircleCI to run unit tests.

.circleci/unittest/linux/scripts/install.sh → .github/scripts/unittest-linux/install.sh

 #!/usr/bin/env bash
 
+# NOTE
+# Currently Linux GPU code has separate run script hardcoded in GHA YAML file.
+# Therefore the CUDA-related things in this script is not used, and it's broken.
+# TODO: Migrate GHA Linux GPU test job to this script.
+
 unset PYTORCH_VERSION
-# For unittest, nightly PyTorch is used as the following section,
-# so no need to set PYTORCH_VERSION.
+# No need to set PYTORCH_VERSION for unit test, as we use nightly PyTorch.
 # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
 
 set -e
 
-root_dir="$(git rev-parse --show-toplevel)"
-conda_dir="${root_dir}/conda"
-env_dir="${root_dir}/env"
-
-cd "${root_dir}"
-
 case "$(uname -s)" in
-    Darwin*) os=MacOSX;;
-    *) os=Linux
+    Darwin*)
+        os=MacOSX;;
+    *)
+        os=Linux
+        eval "$("/opt/conda/bin/conda" shell.bash hook)"
 esac
 
-# 0. Activate conda env
-eval "$("${conda_dir}/bin/conda" shell.bash hook)"
-conda activate "${env_dir}"
-
 # 1. Install PyTorch
 if [ -z "${CUDA_VERSION:-}" ] ; then
     if [ "${os}" == MacOSX ] ; then
@@ -57,6 +54,8 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}"
 )
 
 # 2. Install torchaudio
+conda install --quiet -y 'ffmpeg>=4.1' ninja cmake
+
 printf "* Installing torchaudio\n"
 python setup.py install
 

.circleci/unittest/linux/scripts/run_test.sh → .github/scripts/unittest-linux/run_test.sh

@@ -2,8 +2,7 @@
 
 set -e
 
-eval "$(./conda/bin/conda shell.bash hook)"
-conda activate ./env
+eval "$(/opt/conda/bin/conda shell.bash hook)"
 
 python -m torch.utils.collect_env
 env | grep TORCHAUDIO || true

.github/workflows/unittest-linux-cpu.yml

@@ -54,14 +54,10 @@ jobs:
 
         set -euxo pipefail
 
-        echo '::group::Setup Conda Environment'
-        ./.circleci/unittest/linux/scripts/setup_env.sh
-        echo '::endgroup::'
-
         echo '::group::Install PyTorch and Torchaudio'
-        ./.circleci/unittest/linux/scripts/install.sh
+        ./.github/scripts/unittest-linux/install.sh
         echo '::endgroup::'
 
         echo '::group::Run Tests'
-        ./.circleci/unittest/linux/scripts/run_test.sh
+        ./.github/scripts/unittest-linux/run_test.sh
         echo '::endgroup::'

.github/workflows/unittest-macos-cpu.yml

@@ -53,14 +53,10 @@ jobs:
 
         set -euxo pipefail
 
-        echo '::group::Setup Conda Environment'
-        ./.circleci/unittest/linux/scripts/setup_env.sh
-        echo '::endgroup::'
-
         echo '::group::Install PyTorch and Torchaudio'
-        ./.circleci/unittest/linux/scripts/install.sh
+        ./.github/scripts/unittest-linux/install.sh
         echo '::endgroup::'
 
         echo '::group::Run Tests'
-        ./.circleci/unittest/linux/scripts/run_test.sh
+        ./.github/scripts/unittest-linux/run_test.sh
         echo '::endgroup::'

.github/workflows/unittest-windows-cpu.yml

@@ -47,6 +47,6 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
 
-        ./.circleci/unittest/windows/scripts/setup_env.sh
-        ./.circleci/unittest/windows/scripts/install.sh
-        ./.circleci/unittest/windows/scripts/run_test.sh
+        .github/scripts/unittest-windows/setup_env.sh
+        .github/scripts/unittest-windows/install.sh
+        .github/scripts/unittest-windows/run_test.sh

.github/workflows/unittest-windows-gpu.yml

@@ -48,9 +48,9 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
 
 
-        ./.circleci/unittest/windows/scripts/setup_env.sh
+        .github/scripts/unittest-windows/setup_env.sh
         ./packaging/windows/internal/cuda_install.bat
         ./packaging/windows/internal/driver_update.bat
 
-        ./.circleci/unittest/windows/scripts/install.sh
-        ./.circleci/unittest/windows/scripts/run_test.sh
+        .github/scripts/unittest-windows/install.sh
+        .github/scripts/unittest-windows/run_test.sh