Add base test image with Kaldi featbin and codecs (#581)

* Add test image with Kaldi executable and third_parties

* Use Ubuntu as bese image since pytorch/many-linux is huge

.circleci/config.yml

@@ -236,35 +236,34 @@ jobs:
   unittest_linux:
     <<: *binary_common
     docker:
-      - image: "pytorch/manylinux-cuda100"
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: 2xlarge+
     steps:
       - checkout
       - restore_cache:
 
           keys:
-            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum "./packaging/build_from_source.sh" }}-{{ checksum ".circleci/unittest/environment.yml" }}
+            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}
 
       - run:
           name: Setup
-          command: .circleci/unittest/setup_env.sh
+          command: .circleci/unittest/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum "./packaging/build_from_source.sh" }}-{{ checksum ".circleci/unittest/environment.yml" }}
+          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}
 
           paths:
             - conda
             - env
-            - third_party
       - run:
-          name: Installation
-          command: .circleci/unittest/install.sh
+          name: Install torchaudio
+          command: .circleci/unittest/scripts/install.sh
       - run:
-          name: Run test
-          command: .circleci/unittest/run_test.sh
+          name: Run tests
+          command: .circleci/unittest/scripts/run_test.sh
       - run:
           name: Post process
-          command: .circleci/unittest/post_process.sh
+          command: .circleci/unittest/scripts/post_process.sh
       - store_test_results:
           path: test-results
 

.circleci/config.yml.in

@@ -236,35 +236,34 @@ jobs:
   unittest_linux:
     <<: *binary_common
     docker:
-      - image: "pytorch/manylinux-cuda100"
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: 2xlarge+
     steps:
       - checkout
       - restore_cache:
           {% raw %}
           keys:
-            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum "./packaging/build_from_source.sh" }}-{{ checksum ".circleci/unittest/environment.yml" }}
+            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}
           {% endraw %}
       - run:
           name: Setup
-          command: .circleci/unittest/setup_env.sh
+          command: .circleci/unittest/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum "./packaging/build_from_source.sh" }}-{{ checksum ".circleci/unittest/environment.yml" }}
+          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}
           {% endraw %}
           paths:
             - conda
             - env
-            - third_party
       - run:
-          name: Installation
-          command: .circleci/unittest/install.sh
+          name: Install torchaudio
+          command: .circleci/unittest/scripts/install.sh
       - run:
-          name: Run test
-          command: .circleci/unittest/run_test.sh
+          name: Run tests
+          command: .circleci/unittest/scripts/run_test.sh
       - run:
           name: Post process
-          command: .circleci/unittest/post_process.sh
+          command: .circleci/unittest/scripts/post_process.sh
       - store_test_results:
           path: test-results
 

.circleci/unittest/README.md

+This directory contains;
+
+ - docker
+   Docker image definition and scripts to build and update Docker image for unittest.
+ - scripts
+   Scripts used by CircleCI to run unit tests.

.circleci/unittest/docker/.dockerignore

+*
+!scripts

.circleci/unittest/docker/.gitignore

+scripts/build_third_parties.sh

.circleci/unittest/docker/Dockerfile

+FROM ubuntu:18.04 as builder
+
+RUN apt update -q
+
+################################################################################
+# Build Kaldi
+################################################################################
+RUN apt install -q -y \
+        autoconf \
+        automake \
+        bzip2 \
+        g++ \
+        gfortran \
+        git \
+        libatlas-base-dev \
+        libtool \
+        make \
+        python2.7 \
+        python3 \
+        sox \
+        subversion \
+        unzip \
+        wget \
+        zlib1g-dev
+
+# KALDI uses MKL as a default math library, but we are going to copy featbin binaries and dependent
+# shared libraries to the final image, so we use ATLAS, which is easy to reinstall in the final image.
+RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \
+        cd /opt/kaldi/tools && \
+        make -j $(nproc) && \
+        cd /opt/kaldi/src && \
+        ./configure --shared --mathlib=ATLAS --use-cuda=no && \
+        make featbin -j $(nproc)
+
+# Copy featbins and dependent libraries
+ADD ./scripts /scripts
+RUN bash /scripts/copy_kaldi_executables.sh /opt/kaldi /kaldi
+
+################################################################################
+# Build third party dependencies
+################################################################################
+RUN apt install -q -y curl
+RUN bash /scripts/build_third_parties.sh /
+
+################################################################################
+# Build the final image
+################################################################################
+FROM ubuntu:18.04
+RUN apt update && apt install -y \
+        g++ \
+        gfortran \
+        git \
+        libatlas3-base \
+        wget \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /kaldi /kaldi
+COPY --from=builder /third_party /third_party
+ENV PATH="${PATH}:/kaldi/bin" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/kaldi/lib"

.circleci/unittest/docker/build_and_push.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+
+root_dir="$(git rev-parse --show-toplevel)"
+cp "${root_dir}"/packaging/build_from_source.sh ./scripts/build_third_parties.sh
+
+tag="manylinux"
+image="pytorch/torchaudio_unittest_base:${tag}"
+docker build -t "${image}" .
+docker push "${image}"

.circleci/unittest/docker/scripts/copy_kaldi_executables.sh

+#!/usr/bin/env bash
+
+list_executables() {
+    # List up executables in the given directory
+    find "$1" -type f -executable
+}
+
+list_kaldi_libraries() {
+    # List up shared libraries used by executables found in the given directory ($1)
+    # that reside in Kaldi directory ($2)
+    while read file; do
+        ldd "${file}" | grep -o "${2}.* ";
+    done < <(list_executables "$1") | sort -u
+}
+
+set -euo pipefail
+
+kaldi_root="$(realpath "$1")"
+target_dir="$(realpath "$2")"
+
+bin_dir="${target_dir}/bin"
+lib_dir="${target_dir}/lib"
+
+mkdir -p "${bin_dir}" "${lib_dir}"
+
+# 1. Copy featbins
+printf "Copying executables to %s\n" "${bin_dir}"
+while read file; do
+    printf "  %s\n" "${file}"
+    cp "${file}" "${bin_dir}"
+done < <(list_executables "${kaldi_root}/src/featbin")
+
+# 2. Copy dependent libraries from Kaldi
+printf "Copying libraries to %s\n" "${lib_dir}"
+while read file; do
+    printf "  %s\n" "$file"
+    # If it is not symlink, just copy to the target directory
+    if [ ! -L "${file}" ]; then
+        cp "${file}" "${lib_dir}"
+        continue
+    fi
+
+    # If it is symlink,
+    # 1. Copy the actual library to the target directory.
+    library="$(realpath "${file}")"
+    cp "${library}" "${lib_dir}"
+    # 2. then if the name of the symlink is different from the actual library name,
+    #    create the symlink in the target directory.
+    lib_name="$(basename "${library}")"
+    link_name="$(basename "${file}")"
+    if [ "${lib_name}" != "${link_name}" ]; then
+        printf "    Linking %s -> %s\n" "${lib_name}" "${link_name}"
+        (
+            cd "${lib_dir}"
+            ln -sf "${lib_name}" "${link_name}"
+        )
+    fi
+done < <(list_kaldi_libraries "${bin_dir}" "${kaldi_root}")

.circleci/unittest/install.sh → .circleci/unittest/scripts/install.sh

 #!/usr/bin/env bash
 
 unset PYTORCH_VERSION
-# For unittest, nightly PyTorch is used, and we do not want to fixiate on the version
+# For unittest, nightly PyTorch is used as the following section,
+# so no need to set PYTORCH_VERSION.
+# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
 
 set -e
 
@@ -9,7 +11,9 @@ eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
 printf "* Installing PyTorch nightly build"
-conda install -c pytorch-nightly pytorch cpuonly
+conda install -y -c pytorch-nightly pytorch cpuonly
 
-printf "* Setting up torchaudio\n"
+printf "* Installing torchaudio\n"
+# Link codecs present at /third_party. See Dockerfile for how this is built
+ln -fs /third_party ./third_party
 IS_CONDA=true python setup.py develop

.circleci/unittest/setup_env.sh → .circleci/unittest/scripts/setup_env.sh

 #!/usr/bin/env bash
 
-# This script is for setting up environment for running unit test on CircleCI.
-# To speed up the CI time, the result of environment is cached.
-# PyTorch is not included here, so that it won't be cached.
+# This script is for setting up environment in which unit test is ran.
+# To speed up the CI time, the resulting environment is cached.
+#
+# Do not install PyTorch and torchaudio here, otherwise they also get cached.
 
 set -e
 
@@ -19,21 +20,15 @@ if [ ! -d "${conda_dir}" ]; then
     wget -O miniconda.sh http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
     bash ./miniconda.sh -b -f -p "${conda_dir}"
 fi
-printf "* Checking conda update\n"
 eval "$(${conda_dir}/bin/conda shell.bash hook)"
-conda update -n base -c defaults conda
 
 # 2. Create test environment at ./env
 if [ ! -d "${env_dir}" ]; then
     printf "* Creating a test environment\n"
     conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
 fi
-printf "* Installing dependencies (except PyTorch)\n"
 conda activate "${env_dir}"
-conda env update --file "${this_dir}/environment.yml" --prune
 
-# 3. Build codecs at ./third_party
-if [ ! -d "./third_party" ]; then
-    printf "* Building Codecs"
-    ./packaging/build_from_source.sh "$PWD"
-fi
+# 3. Install Conda dependencies
+printf "* Installing dependencies (except PyTorch)\n"
+conda env update --file "${this_dir}/environment.yml" --prune