Switch to Nova MacOS Wheel (#2907)

Summary:
Switch to Nova MacOS and M1 Wheels. This PR is a step in migrating from CircleCI to the Nova workflow.

- [x] Disable the CircleCI builds for MacOS Wheel.
- [x] Disable the CircleCI builds for M1 Wheel.
- [x] Enable the Nova workflow for MacOS Wheel.
- [x] Enable the Nova workflow for M1 Wheel.

Pull Request resolved: https://github.com/pytorch/audio/pull/2907

Reviewed By: osalpekar, mthrok

Differential Revision: D42040965

Pulled By: DanilBaibak

fbshipit-source-id: b87f028cf5686bf97265109591fb0a8c1190324c

.circleci/config.yml

@@ -929,30 +929,6 @@ workflows:
           python_version: foo
           requires:
           - download_third_parties
-      - binary_macos_wheel:
-          cuda_version: cpu
-          name: binary_macos_wheel_py3.7_cpu
-          python_version: '3.7'
-          requires:
-          - build_ffmpeg_macos
-      - binary_macos_wheel:
-          cuda_version: cpu
-          name: binary_macos_wheel_py3.8_cpu
-          python_version: '3.8'
-          requires:
-          - build_ffmpeg_macos
-      - binary_macos_wheel:
-          cuda_version: cpu
-          name: binary_macos_wheel_py3.9_cpu
-          python_version: '3.9'
-          requires:
-          - build_ffmpeg_macos
-      - binary_macos_wheel:
-          cuda_version: cpu
-          name: binary_macos_wheel_py3.10_cpu
-          python_version: '3.10'
-          requires:
-          - build_ffmpeg_macos
       - binary_windows_wheel:
           cuda_version: cpu
           name: binary_windows_wheel_py3.7_cpu
@@ -1590,102 +1566,6 @@ workflows:
           python_version: foo
           requires:
           - download_third_parties
-      - binary_macos_wheel:
-          cuda_version: cpu
-          filters:
-            branches:
-              only:
-              - nightly
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
-          name: nightly_binary_macos_wheel_py3.7_cpu
-          python_version: '3.7'
-          requires:
-          - build_ffmpeg_macos
-      - binary_wheel_upload:
-          context: org-member
-          filters:
-            branches:
-              only:
-              - nightly
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
-          name: nightly_binary_macos_wheel_py3.7_cpu_upload
-          requires:
-          - nightly_binary_macos_wheel_py3.7_cpu
-          subfolder: ''
-      - binary_macos_wheel:
-          cuda_version: cpu
-          filters:
-            branches:
-              only:
-              - nightly
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
-          name: nightly_binary_macos_wheel_py3.8_cpu
-          python_version: '3.8'
-          requires:
-          - build_ffmpeg_macos
-      - binary_wheel_upload:
-          context: org-member
-          filters:
-            branches:
-              only:
-              - nightly
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
-          name: nightly_binary_macos_wheel_py3.8_cpu_upload
-          requires:
-          - nightly_binary_macos_wheel_py3.8_cpu
-          subfolder: ''
-      - binary_macos_wheel:
-          cuda_version: cpu
-          filters:
-            branches:
-              only:
-              - nightly
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
-          name: nightly_binary_macos_wheel_py3.9_cpu
-          python_version: '3.9'
-          requires:
-          - build_ffmpeg_macos
-      - binary_wheel_upload:
-          context: org-member
-          filters:
-            branches:
-              only:
-              - nightly
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
-          name: nightly_binary_macos_wheel_py3.9_cpu_upload
-          requires:
-          - nightly_binary_macos_wheel_py3.9_cpu
-          subfolder: ''
-      - binary_macos_wheel:
-          cuda_version: cpu
-          filters:
-            branches:
-              only:
-              - nightly
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
-          name: nightly_binary_macos_wheel_py3.10_cpu
-          python_version: '3.10'
-          requires:
-          - build_ffmpeg_macos
-      - binary_wheel_upload:
-          context: org-member
-          filters:
-            branches:
-              only:
-              - nightly
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
-          name: nightly_binary_macos_wheel_py3.10_cpu_upload
-          requires:
-          - nightly_binary_macos_wheel_py3.10_cpu
-          subfolder: ''
       - binary_windows_wheel:
           cuda_version: cpu
           filters:

.circleci/regenerate.py

@@ -42,10 +42,13 @@ def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6):
             for python_version in PYTHON_VERSIONS:
                 for cu_version in CU_VERSIONS_DICT[os_type]:
                     fb = filter_branch
-                    if (cu_version.startswith("rocm") and btype == "conda") or (
-                        os_type == "linux" and btype == "wheel"
+                    if (
+                        (cu_version.startswith("rocm") and btype == "conda")
+                        or (os_type == "linux" and btype == "wheel")
+                        or (os_type == "macos" and btype == "wheel")
                     ):
                         continue
+
                     if not fb and (
                         os_type == "linux" and btype == "wheel" and python_version == "3.8" and cu_version == "cpu"
                     ):

.github/workflows/build-m1-binaries.yml

@@ -15,88 +15,6 @@ on:
 env:
   CHANNEL: "nightly"
 jobs:
-  build_wheels:
-    name: "Build TorchAudio M1 wheels"
-    runs-on: macos-m1-12
-    strategy:
-      matrix:
-        py_vers: [ "3.8", "3.9", "3.10" ]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-      - name: Set CHANNEL (only for tagged pushes)
-        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }}
-        run: |
-          # reference ends with an RC suffix
-          if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
-            echo "CHANNEL=test" >> "$GITHUB_ENV"
-          fi
-      - name: Set Release CHANNEL (for release)
-        if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }}
-        run: |
-          echo "CHANNEL=test" >> "$GITHUB_ENV"
-      - name: Setup miniconda
-        uses: pytorch/test-infra/.github/actions/setup-miniconda@main
-      - name: Build TorchAudio M1 wheel
-        shell: arch -arch arm64 bash {0}
-        env:
-          ENV_NAME: conda-env-${{ github.run_id }}
-          PY_VERS: ${{ matrix.py_vers }}
-          CU_VERSION: cpu
-          USE_FFMPEG: true
-          USE_OPENMP: false
-          USE_CUDA: false
-          # Needed so that delocate puts files in places it can actually modify
-          TMPDIR: ${{ runner.temp }}
-        run: |
-          set -ex
-          . packaging/pkg_helpers.bash
-          # if we are uploading to test channell, our version consist only of the base: 0.x.x - no date string or suffix added
-          if [[ $CHANNEL == "test" ]]; then
-            setup_base_build_version
-          else
-            setup_build_version
-          fi
-          git submodule update --init --recursive
-          export SOURCE_ROOT_DIR=$(pwd)
-          export FFMPEG_ROOT="${SOURCE_ROOT_DIR}/third_party/ffmpeg"
-          . packaging/ffmpeg/build.sh
-          conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja wheel pkg-config
-          conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL}
-          conda run -p ${ENV_NAME} python3 -mpip install delocate
-          conda run -p ${ENV_NAME} python3 setup.py bdist_wheel
-          export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')"
-          conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/*.whl
-          conda env remove -p ${ENV_NAME}
-      - name: Test wheel
-        shell: arch -arch arm64 bash {0}
-        env:
-          ENV_NAME: conda-test-env-${{ github.run_id }}
-          PY_VERS: ${{ matrix.py_vers }}
-        run: |
-          set -ex
-          conda create -yp ${ENV_NAME} python=${PY_VERS} numpy
-          conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL}
-          conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl
-          # Test torch is importable, by changing cwd and running import commands
-          conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchaudio;print('torchaudio version is ', torchaudio.__version__)"
-          conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchaudio;torchaudio.set_audio_backend('sox_io')"
-          conda env remove -p ${ENV_NAME}
-      - name: Upload wheel to GitHub
-        uses: actions/upload-artifact@v3
-        with:
-          name: torchaudio-py${{ matrix.py_vers }}-macos11-m1
-          path: dist/
-      - name: Upload wheel to S3
-        if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }}
-        shell: arch -arch arm64 bash {0}
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
-        run: |
-          for pkg in dist/*; do
-            aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read
-          done
   build_conda:
     name: "Build TorchAudio M1 conda packages"
     runs-on: macos-m1-12

.github/workflows/build-wheels-m1.yml

@@ -39,9 +39,7 @@ jobs:
       smoke-test-script: ${{ matrix.smoke-test-script }}
       runner-type: macos-m1-12
       package-name: ${{ matrix.package-name }}
-      # Using "development" as trigger event so these binaries are not uploaded
-      # to official channels yet
-      trigger-event: development
+      trigger-event: ${{ github.event_name }}
     secrets:
       AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
       AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}

.github/workflows/build_wheels_macos.yml

@@ -39,9 +39,7 @@ jobs:
       smoke-test-script: ${{ matrix.smoke-test-script }}
       runner-type: macos-12
       package-name: ${{ matrix.package-name }}
-      # Using "development" as trigger event so these binaries are not uploaded
-      # to official channels yet
-      trigger-event: development
+      trigger-event: ${{ github.event_name }}
     secrets:
       AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
       AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}