Limit the number of downloads par workflow for reliable build (#670)

* Add caching to binary build for more reliable build

* Apply same logic to unittest workflow

.circleci/config.yml

@@ -60,6 +60,34 @@ jobs:
             python .circleci/regenerate.py
             git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
 
+  download_third_parties_nix:
+    docker:
+      - image: "centos:8"
+    resource_class: small
+    steps:
+      - checkout
+      - run:
+          name: Generate cache key
+          # This will refresh cache on daily
+          command: echo "$(date +"%Y-%U-%d")" > .circleci-daily
+      - restore_cache:
+
+          keys:
+            - tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+
+      - run:
+          command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
+      - save_cache:
+
+          key: tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+
+          paths:
+            - third_party/tmp
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - tmp
+
   binary_linux_wheel:
     <<: *binary_common
     docker:
@@ -67,6 +95,8 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run: packaging/build_wheel.sh
       - store_artifacts:
           path: dist
@@ -82,6 +112,8 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run: packaging/build_conda.sh
       - store_artifacts:
           path: /opt/conda/conda-bld/linux-64
@@ -96,6 +128,8 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run:
           # Cannot easily deduplicate this as source'ing activate
           # will set environment variables which we need to propagate
@@ -118,6 +152,8 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run:
           command: |
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -322,6 +358,8 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -363,6 +401,8 @@ jobs:
       image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -506,24 +546,38 @@ workflows:
   build:
     jobs:
       - circleci_consistency
+      - download_third_parties_nix:
+          name: download_third_parties_nix
       - binary_linux_wheel:
           name: binary_linux_wheel_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - binary_linux_wheel:
           name: binary_linux_wheel_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - binary_linux_wheel:
           name: binary_linux_wheel_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - binary_macos_wheel:
           name: binary_macos_wheel_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - binary_macos_wheel:
           name: binary_macos_wheel_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - binary_macos_wheel:
           name: binary_macos_wheel_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - binary_windows_wheel:
           name: binary_windows_wheel_py3.6
           python_version: '3.6'
@@ -536,21 +590,33 @@ workflows:
       - binary_linux_conda:
           name: binary_linux_conda_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - binary_linux_conda:
           name: binary_linux_conda_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - binary_linux_conda:
           name: binary_linux_conda_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - binary_macos_conda:
           name: binary_macos_conda_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - binary_macos_conda:
           name: binary_macos_conda_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - binary_macos_conda:
           name: binary_macos_conda_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - binary_windows_conda:
           name: binary_windows_conda_py3.6
           python_version: '3.6'
@@ -562,36 +628,50 @@ workflows:
           python_version: '3.8'
   unittest:
     jobs:
+      - download_third_parties_nix:
+          name: download_third_parties_nix
       - unittest_linux_cpu:
           name: unittest_linux_cpu_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - stylecheck:
           name: stylecheck_py3.6
           python_version: '3.6'
       - unittest_linux_cpu:
           name: unittest_linux_cpu_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - unittest_linux_cpu:
           name: unittest_linux_cpu_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - unittest_linux_gpu:
           filters:
             branches:
               only: master
           name: unittest_linux_gpu_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - unittest_linux_gpu:
           filters:
             branches:
               only: master
           name: unittest_linux_gpu_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - unittest_linux_gpu:
           filters:
             branches:
               only: master
           name: unittest_linux_gpu_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - unittest_windows_cpu:
           name: unittest_windows_cpu_py3.6
           python_version: '3.6'
@@ -625,12 +705,19 @@ workflows:
           filters:
             branches:
               only: nightly
+      - download_third_parties_nix:
+          filters:
+            branches:
+              only: nightly
+          name: download_third_parties_nix
       - binary_linux_wheel:
           filters:
             branches:
               only: nightly
           name: nightly_binary_linux_wheel_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - binary_wheel_upload:
           context: org-member
           filters:
@@ -653,6 +740,8 @@ workflows:
               only: nightly
           name: nightly_binary_linux_wheel_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - binary_wheel_upload:
           context: org-member
           filters:
@@ -675,6 +764,8 @@ workflows:
               only: nightly
           name: nightly_binary_linux_wheel_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - binary_wheel_upload:
           context: org-member
           filters:
@@ -697,6 +788,8 @@ workflows:
               only: nightly
           name: nightly_binary_macos_wheel_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - binary_wheel_upload:
           context: org-member
           filters:
@@ -711,6 +804,8 @@ workflows:
               only: nightly
           name: nightly_binary_macos_wheel_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - binary_wheel_upload:
           context: org-member
           filters:
@@ -725,6 +820,8 @@ workflows:
               only: nightly
           name: nightly_binary_macos_wheel_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - binary_wheel_upload:
           context: org-member
           filters:
@@ -805,6 +902,8 @@ workflows:
               only: nightly
           name: nightly_binary_linux_conda_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - binary_conda_upload:
           context: org-member
           filters:
@@ -827,6 +926,8 @@ workflows:
               only: nightly
           name: nightly_binary_linux_conda_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - binary_conda_upload:
           context: org-member
           filters:
@@ -849,6 +950,8 @@ workflows:
               only: nightly
           name: nightly_binary_linux_conda_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - binary_conda_upload:
           context: org-member
           filters:
@@ -871,6 +974,8 @@ workflows:
               only: nightly
           name: nightly_binary_macos_conda_py3.6
           python_version: '3.6'
+          requires:
+          - download_third_parties_nix
       - binary_conda_upload:
           context: org-member
           filters:
@@ -885,6 +990,8 @@ workflows:
               only: nightly
           name: nightly_binary_macos_conda_py3.7
           python_version: '3.7'
+          requires:
+          - download_third_parties_nix
       - binary_conda_upload:
           context: org-member
           filters:
@@ -899,6 +1006,8 @@ workflows:
               only: nightly
           name: nightly_binary_macos_conda_py3.8
           python_version: '3.8'
+          requires:
+          - download_third_parties_nix
       - binary_conda_upload:
           context: org-member
           filters:

.circleci/config.yml.in

@@ -60,6 +60,34 @@ jobs:
             python .circleci/regenerate.py
             git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
 
+  download_third_parties_nix:
+    docker:
+      - image: "centos:8"
+    resource_class: small
+    steps:
+      - checkout
+      - run:
+          name: Generate cache key
+          # This will refresh cache on daily
+          command: echo "$(date +"%Y-%U-%d")" > .circleci-daily
+      - restore_cache:
+          {% raw %}
+          keys:
+            - tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+          {% endraw %}
+      - run:
+          command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
+      - save_cache:
+          {% raw %}
+          key: tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+          {% endraw %}
+          paths:
+            - third_party/tmp
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - tmp
+
   binary_linux_wheel:
     <<: *binary_common
     docker:
@@ -67,6 +95,8 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run: packaging/build_wheel.sh
       - store_artifacts:
           path: dist
@@ -82,6 +112,8 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run: packaging/build_conda.sh
       - store_artifacts:
           path: /opt/conda/conda-bld/linux-64
@@ -96,6 +128,8 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run:
           # Cannot easily deduplicate this as source'ing activate
           # will set environment variables which we need to propagate
@@ -118,6 +152,8 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run:
           command: |
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -322,6 +358,8 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -363,6 +401,8 @@ jobs:
       image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
     steps:
       - checkout
+      - attach_workspace:
+          at: third_party
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.

.circleci/regenerate.py

@@ -24,6 +24,7 @@ PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]
 
 def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
     w = []
+    w += build_download_job(filter_branch)
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos", "windows"]:
             for python_version in PYTHON_VERSIONS:
@@ -32,6 +33,16 @@ def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
     return indent(indentation, w)
 
 
+def build_download_job(filter_branch):
+    job = {
+        "name": "download_third_parties_nix",
+    }
+
+    if filter_branch:
+        job["filters"] = gen_filter_branch_tree(filter_branch)
+    return [{"download_third_parties_nix": job}]
+
+
 def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False):
 
     w = []
@@ -64,6 +75,9 @@ def generate_base_workflow(base_workflow_name, python_version, filter_branch, os
         "python_version": python_version,
     }
 
+    if os_type in ['linux', 'macos']:
+        d['requires'] = ['download_third_parties_nix']
+
     if filter_branch:
         d["filters"] = gen_filter_branch_tree(filter_branch)
 
@@ -112,6 +126,7 @@ def indent(indentation, data_list):
 
 def unittest_workflows(indentation=6):
     jobs = []
+    jobs += build_download_job(None)
     for os_type in ["linux", "windows"]:
         for device_type in ["cpu", "gpu"]:
             for i, python_version in enumerate(PYTHON_VERSIONS):
@@ -122,6 +137,10 @@ def unittest_workflows(indentation=6):
 
                 if device_type == 'gpu':
                     job['filters'] = gen_filter_branch_tree('master')
+
+                if os_type != "windows":
+                    job['requires'] = ['download_third_parties_nix']
+
                 jobs.append({f"unittest_{os_type}_{device_type}": job})
 
                 if i == 0 and os_type == "linux" and device_type == "cpu":

build_tools/setup_helpers/build_third_party.sh

 #!/bin/bash
-# Build third party libraries in `<repo_root>/third_party/build` or in `<given_prefix>/third_party/build`.
+# Build third party libraries (SoX, lame, libmad, and flac)
+# Usage: ./build_thid_parth.sh [prefix] [download_only?=false]
 
 set -e
 
@@ -10,6 +11,7 @@ prefix="${1:-}"
 if [ -z "${prefix}" ]; then
     prefix="${root_dir}"
 fi
+download_only="${2:-false}"
 
 tp_dir="${prefix}/third_party"
 tmp_dir="${tp_dir}/tmp"
@@ -20,17 +22,29 @@ mkdir -p "${tmp_dir}" "${build_dir}"
 . "${this_dir}/build_third_party_helper.sh"
 
 if ! found_lame "${build_dir}" ; then
-   build_lame "${tmp_dir}" "${build_dir}"
+    get_lame "${tmp_dir}"
+    if [ "${download_only}" = "false" ]; then
+        build_lame "${tmp_dir}" "${build_dir}"
+    fi
 fi
 
 if ! found_flac "${build_dir}" ; then
-    build_flac "${tmp_dir}" "${build_dir}"
+   get_flac "${tmp_dir}"
+   if [ "${download_only}" = "false" ]; then
+       build_flac "${tmp_dir}" "${build_dir}"
+   fi
 fi
 
 if ! found_mad "${build_dir}" ; then
-    build_mad "${tmp_dir}" "${build_dir}"
+   get_mad "${tmp_dir}"
+   if [ "${download_only}" = "false" ]; then
+       build_mad "${tmp_dir}" "${build_dir}"
+   fi
 fi
 
 if ! found_sox "${build_dir}" ; then
-    build_sox "${tmp_dir}" "${build_dir}"
+   get_sox "${tmp_dir}"
+   if [ "${download_only}" = "false" ]; then
+       build_sox "${tmp_dir}" "${build_dir}"
+   fi
 fi

build_tools/setup_helpers/build_third_party_helper.sh

@@ -57,23 +57,34 @@ found_sox() {
     all_found "$1" 'include/sox.h' 'lib/libsox.a'
 }
 
-build_lame() {
+LAME="lame-3.99.5"
+LAME_ARCHIVE="${LAME}.tar.gz"
+
+get_lame() {
     work_dir="$1"
-    install_dir="$2"
-    package="lame-3.99.5"
-    url="https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz"
+    url="https://downloads.sourceforge.net/project/lame/lame/3.99/${LAME_ARCHIVE}"
     (
         cd "${work_dir}"
-        if [ ! -d "${package}" ]; then
-            if [ ! -f "${package}.tar.gz" ]; then
+        if [ ! -d "${LAME}" ]; then
+            if [ ! -f "${LAME_ARCHIVE}" ]; then
                 printf "Fetching liblame from %s\n" "${url}"
-                curl $CURL_OPTS -o "${package}.tar.gz" "${url}"
+                curl $CURL_OPTS -O "${url}"
             fi
-            tar xfp "${package}.tar.gz"
         fi
+    )
+}
+
+build_lame() {
+    work_dir="$1"
+    install_dir="$2"
+    (
+        cd "${work_dir}"
+        if [ ! -d "${LAME}" ]; then
+            tar xfp "${LAME_ARCHIVE}"
+        fi
+        cd "${LAME}"
         # build statically
         printf "Building liblame\n"
-        cd "${package}"
         if [ ! -f Makefile ]; then
             ./configure ${CONFIG_OPTS} \
                         --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
@@ -84,23 +95,34 @@ build_lame() {
     )
 }
 
-build_flac() {
+FLAC="flac-1.3.2"
+FLAC_ARCHIVE="${FLAC}.tar.xz"
+
+get_flac() {
     work_dir="$1"
-    install_dir="$2"
-    package="flac-1.3.2"
-    url="https://downloads.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz"
+    url="https://downloads.sourceforge.net/project/flac/flac-src/${FLAC_ARCHIVE}"
     (
         cd "${work_dir}"
-        if [ ! -d "${package}" ]; then
-            if [ ! -f "${package}.tar.xz" ]; then
+        if [ ! -d "${FLAC}" ]; then
+            if [ ! -f "${FLAC_ARCHIVE}" ]; then
                 printf "Fetching flac from %s\n" "${url}"
-                curl $CURL_OPTS -o "${package}.tar.xz" "${url}"
+                curl $CURL_OPTS -O "${url}"
             fi
-            tar xfp "${package}.tar.xz"
         fi
+    )
+}
+
+build_flac() {
+    work_dir="$1"
+    install_dir="$2"
+    (
+        cd "${work_dir}"
+        if [ ! -d "${FLAC}" ]; then
+            tar xfp "${FLAC_ARCHIVE}"
+        fi
+        cd "${FLAC}"
         # build statically
         printf "Building flac\n"
-        cd "${package}"
         if [ ! -f Makefile ]; then
             ./configure ${CONFIG_OPTS} \
                         --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
@@ -111,23 +133,34 @@ build_flac() {
     )
 }
 
-build_mad() {
+LIBMAD="libmad-0.15.1b"
+LIBMAD_ARCHIVE="${LIBMAD}.tar.gz"
+
+get_mad() {
     work_dir="$1"
-    install_dir="$2"
-    package="libmad-0.15.1b"
-    url="https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz"
+    url="https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/${LIBMAD_ARCHIVE}"
     (
         cd "${work_dir}"
-        if [ ! -d "${package}" ]; then
-            if [ ! -f "${package}.tar.gz" ]; then
+        if [ ! -d "${LIBMAD}" ]; then
+            if [ ! -f "${LIBMAD_ARCHIVE}" ]; then
                 printf "Fetching mad from %s\n" "${url}"
-                curl $CURL_OPTS -o "${package}.tar.gz" "${url}"
+                curl $CURL_OPTS -O "${url}"
             fi
-            tar xfp "${package}.tar.gz"
         fi
+    )
+}
+
+build_mad() {
+    work_dir="$1"
+    install_dir="$2"
+    (
+        cd "${work_dir}"
+        if [ ! -d "${LIBMAD}" ]; then
+            tar xfp "${LIBMAD_ARCHIVE}"
+        fi
+        cd "${LIBMAD}"
         # build statically
         printf "Building mad\n"
-        cd "${package}"
         if [ ! -f Makefile ]; then
             # See https://stackoverflow.com/a/12864879/23845
             sed -i.bak 's/-march=i486//' configure
@@ -140,23 +173,34 @@ build_mad() {
     )
 }
 
-build_sox() {
+SOX="sox-14.4.2"
+SOX_ARCHIVE="${SOX}.tar.bz2"
+
+get_sox() {
     work_dir="$1"
-    install_dir="$2"
-    package="sox-14.4.2"
-    url="https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2"
+    url="https://downloads.sourceforge.net/project/sox/sox/14.4.2/${SOX_ARCHIVE}"
     (
         cd "${work_dir}"
-        if [ ! -d "${package}" ]; then
-            if [ ! -f "${package}.tar.bz2" ]; then
+        if [ ! -d "${SOX}" ]; then
+            if [ ! -f "${SOX_ARCHIVE}" ]; then
                 printf "Fetching SoX from %s\n" "${url}"
-                curl $CURL_OPTS -o "${package}.tar.bz2" "${url}"
+                curl $CURL_OPTS -O "${url}"
             fi
-            tar xfp "${package}.tar.bz2"
         fi
+    )
+}
+
+build_sox() {
+    work_dir="$1"
+    install_dir="$2"
+    (
+        cd "${work_dir}"
+        if [ ! -d "${SOX}" ]; then
+            tar xfp "${SOX_ARCHIVE}"
+        fi
+        cd "${SOX}"
         # build statically
-        printf "Building sox\n"
-        cd "${package}"
+        printf "Building SoX\n"
         if [ ! -f Makefile ]; then
             # --without-png makes OS X build less hazardous; somehow the build
             # finds png and enables it.  We don't want it; we'd need to package