Skip to content

GitLab

Commit cc89f743 authored by Omkar Salpekar's avatar Omkar Salpekar Committed by Facebook GitHub Bot
Browse files

[Nova] Deleting Remaining CircleCI jobs (#3399) 

Summary:
This job completely deletes the CircleCI `config.yml`. Here is what was remaining in the config at the point of deletion:

Used Jobs:
* **Lint** - Now running on Nova - see https://github.com/pytorch/audio/actions/runs/5144082942 for an example run on the latest PR in trunk
* **CircleCI Consistency** - Not needed anymore now if there is no CCI config.

Unused Jobs:
* **build-ffmpeg-$OS** - For the build jobs, we are already building FFMPEG from source as part of the Nova workflows.
* **download-third-parties** - This is caching. We currently do not have caching in Nova jobs, but atalman is working on adding support for this as a future optimization.

Pull Request resolved: https://github.com/pytorch/audio/pull/3399

Reviewed By: mthrok

Differential Revision: D46363921

Pulled By: osalpekar

fbshipit-source-id: 8abf5b0c1612c3492908fb2f5797e6b0a3c70766
parent c7ac1aff
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 626 deletions
.circleci/config.yml deleted 100644 → 0
View file @ c7ac1aff
version: 2.1
# How to test the Linux jobs:
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8
# - Replace binary_linux_wheel_py3.8 with the name of the job you want to test.
# Job names are 'name:' key.
executors:
windows-cpu:
machine:
resource_class: windows.xlarge
image: windows-server-2019-vs2019:stable
shell: bash.exe
windows-gpu:
machine:
resource_class: windows.gpu.nvidia.medium
image: windows-server-2019-nvidia:stable
shell: bash.exe
commands:
generate_cache_key:
description: "Generates a cache key file that changes daily"
steps:
- run:
name: Generate cache key
command: echo "$(date +"%Y-%m-%d")" > .cachekey
designate_upload_channel:
description: "inserts the correct upload channel into ${BASH_ENV}"
steps:
- run:
name: adding UPLOAD_CHANNEL to BASH_ENV
command: |
our_upload_channel=nightly
# On tags upload to test instead
if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then
our_upload_channel=test
fi
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
load_conda_channel_flags:
description: "Determines whether we need extra conda channels"
steps:
- run:
name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV
command: |
CONDA_CHANNEL_FLAGS=""
# formerly used to add conda-forge flags for Python 3.9, reserving the mechanism for future python upgrades
windows_install_cuda:
description: "Install desired CUDA version on Windows runners"
steps:
- run:
name: Install CUDA
command: |
packaging/windows/internal/cuda_install.bat
binary_common: &binary_common
parameters:
# Edit these defaults to do a release
build_version:
description: "version number of release binary; by default, build a nightly"
type: string
default: ""
pytorch_version:
description: "PyTorch version to build against; by default, use a nightly"
type: string
default: ""
# Don't edit these
python_version:
description: "Python version to build against (e.g., 3.8)"
type: string
cuda_version:
description: "CUDA version to build against (e.g., cpu, cu101)"
type: string
default: "cpu"
wheel_docker_image:
description: "Wheel only: what docker image to use"
type: string
default: "pytorch/manylinux-cuda116"
conda_docker_image:
description: "Conda only: what docker image to use"
type: string
default: "pytorch/conda-builder:cuda116"
environment: &environment
PYTHON_VERSION: << parameters.python_version >>
BUILD_VERSION: << parameters.build_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
CU_VERSION: << parameters.cuda_version >>
MACOSX_DEPLOYMENT_TARGET: 10.9
smoke_test_common: &smoke_test_common
<<: *binary_common
docker:
- image: pytorch/torchaudio_unittest_base:smoke_test-20220425
resource_class: large
jobs:
circleci_consistency:
docker:
- image: cimg/python:3.8
steps:
- checkout
- run:
command: |
pip install --user --progress-bar off jinja2 pyyaml
python .circleci/regenerate.py
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
lint_python_and_config:
docker:
- image: circleci/python:3.8
steps:
- checkout
- run:
name: Install pre-commit
command: pip install --user --progress-bar off pre-commit
- run:
name: Install pre-commit hooks
command: pre-commit install-hooks
- run:
name: Lint Python code and config files
command: pre-commit run --all-files
- run:
name: Required lint modifications
when: always
command: git --no-pager diff --color=always
download_third_parties:
docker:
- image: "pytorch/torchaudio_unittest_base:manylinux"
resource_class: small
steps:
- checkout
- generate_cache_key
- restore_cache:
keys:
- tp-nix-v2-{{ checksum ".cachekey" }}
- run:
command: |
mkdir -p third_party/archives/
wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/*/CMakeLists.txt)
- save_cache:
key: tp-nix-v2-{{ checksum ".cachekey" }}
paths:
- third_party/archives
- persist_to_workspace:
root: third_party
paths:
- archives
build_ffmpeg_linux:
<<: *binary_common
docker:
- image: << parameters.wheel_docker_image >>
resource_class: 2xlarge+
steps:
- checkout
- generate_cache_key
- restore_cache:
keys:
- ffmpeg-linux-v0-{{ checksum ".cachekey" }}
- run:
command: |
export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
if [[ ! -d ${FFMPEG_ROOT} ]]; then
packaging/ffmpeg/build.sh
fi
- save_cache:
key: ffmpeg-linux-v0-{{ checksum ".cachekey" }}
paths:
- third_party/ffmpeg
- persist_to_workspace:
root: third_party
paths:
- ffmpeg
build_ffmpeg_macos:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout
- generate_cache_key
- restore_cache:
keys:
- ffmpeg-macos-v0-{{ checksum ".cachekey" }}
- run:
command: |
export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
if [[ ! -d ${FFMPEG_ROOT} ]]; then
packaging/ffmpeg/build.sh
fi
- save_cache:
key: ffmpeg-macos-v0-{{ checksum ".cachekey" }}
paths:
- third_party/ffmpeg
- persist_to_workspace:
root: third_party
paths:
- ffmpeg
build_ffmpeg_windows:
<<: *binary_common
machine:
resource_class: windows.xlarge
image: windows-server-2019-vs2019:stable
# Note:
# Unlike other Windows job, this job uses cmd.exe as shell because
# we need to invoke bash.exe from msys2 in ffmpeg build process, and doing so
# from different installation of bash.exe (the one from the VM) cause issue
shell: cmd.exe
steps:
- checkout
- run: date /t > .cachekey
- restore_cache:
keys:
- ffmpeg-windows-{{ checksum ".cachekey" }}
- run: packaging\ffmpeg\build.bat
- save_cache:
key: ffmpeg-windows-{{ checksum ".cachekey" }}
paths:
- third_party/ffmpeg
- persist_to_workspace:
root: third_party
paths:
- ffmpeg
workflows:
lint:
jobs:
- lint_python_and_config
unittest:
jobs:
- circleci_consistency
.circleci/config.yml.in deleted 100644 → 0
View file @ c7ac1aff
version: 2.1
# How to test the Linux jobs:
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8
# - Replace binary_linux_wheel_py3.8 with the name of the job you want to test.
# Job names are 'name:' key.
executors:
windows-cpu:
machine:
resource_class: windows.xlarge
image: windows-server-2019-vs2019:stable
shell: bash.exe
windows-gpu:
machine:
resource_class: windows.gpu.nvidia.medium
image: windows-server-2019-nvidia:stable
shell: bash.exe
commands:
generate_cache_key:
description: "Generates a cache key file that changes daily"
steps:
- run:
name: Generate cache key
command: echo "$(date +"%Y-%m-%d")" > .cachekey
designate_upload_channel:
description: "inserts the correct upload channel into ${BASH_ENV}"
steps:
- run:
name: adding UPLOAD_CHANNEL to BASH_ENV
command: |
our_upload_channel=nightly
# On tags upload to test instead
if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then
our_upload_channel=test
fi
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
load_conda_channel_flags:
description: "Determines whether we need extra conda channels"
steps:
- run:
name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV
command: |
CONDA_CHANNEL_FLAGS=""
# formerly used to add conda-forge flags for Python 3.9, reserving the mechanism for future python upgrades
windows_install_cuda:
description: "Install desired CUDA version on Windows runners"
steps:
- run:
name: Install CUDA
command: |
packaging/windows/internal/cuda_install.bat
binary_common: &binary_common
parameters:
# Edit these defaults to do a release
build_version:
description: "version number of release binary; by default, build a nightly"
type: string
default: ""
pytorch_version:
description: "PyTorch version to build against; by default, use a nightly"
type: string
default: ""
# Don't edit these
python_version:
description: "Python version to build against (e.g., 3.8)"
type: string
cuda_version:
description: "CUDA version to build against (e.g., cpu, cu101)"
type: string
default: "cpu"
wheel_docker_image:
description: "Wheel only: what docker image to use"
type: string
default: "pytorch/manylinux-cuda116"
conda_docker_image:
description: "Conda only: what docker image to use"
type: string
default: "pytorch/conda-builder:cuda116"
environment: &environment
PYTHON_VERSION: << parameters.python_version >>
BUILD_VERSION: << parameters.build_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
CU_VERSION: << parameters.cuda_version >>
MACOSX_DEPLOYMENT_TARGET: 10.9
smoke_test_common: &smoke_test_common
<<: *binary_common
docker:
- image: pytorch/torchaudio_unittest_base:smoke_test-20220425
resource_class: large
jobs:
circleci_consistency:
docker:
- image: cimg/python:3.8
steps:
- checkout
- run:
command: |
pip install --user --progress-bar off jinja2 pyyaml
python .circleci/regenerate.py
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
lint_python_and_config:
docker:
- image: circleci/python:3.8
steps:
- checkout
- run:
name: Install pre-commit
command: pip install --user --progress-bar off pre-commit
- run:
name: Install pre-commit hooks
command: pre-commit install-hooks
- run:
name: Lint Python code and config files
command: pre-commit run --all-files
- run:
name: Required lint modifications
when: always
command: git --no-pager diff --color=always
download_third_parties:
docker:
- image: "pytorch/torchaudio_unittest_base:manylinux"
resource_class: small
steps:
- checkout
- generate_cache_key
- restore_cache:
{% raw %}
keys:
- tp-nix-v2-{{ checksum ".cachekey" }}
{% endraw %}
- run:
command: |
mkdir -p third_party/archives/
wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/*/CMakeLists.txt)
- save_cache:
{% raw %}
key: tp-nix-v2-{{ checksum ".cachekey" }}
{% endraw %}
paths:
- third_party/archives
- persist_to_workspace:
root: third_party
paths:
- archives
build_ffmpeg_linux:
<<: *binary_common
docker:
- image: << parameters.wheel_docker_image >>
resource_class: 2xlarge+
steps:
- checkout
- generate_cache_key
- restore_cache:
{% raw %}
keys:
- ffmpeg-linux-v0-{{ checksum ".cachekey" }}
{% endraw %}
- run:
command: |
export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
if [[ ! -d ${FFMPEG_ROOT} ]]; then
packaging/ffmpeg/build.sh
fi
- save_cache:
{% raw %}
key: ffmpeg-linux-v0-{{ checksum ".cachekey" }}
{% endraw %}
paths:
- third_party/ffmpeg
- persist_to_workspace:
root: third_party
paths:
- ffmpeg
build_ffmpeg_macos:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout
- generate_cache_key
- restore_cache:
{% raw %}
keys:
- ffmpeg-macos-v0-{{ checksum ".cachekey" }}
{% endraw %}
- run:
command: |
export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
if [[ ! -d ${FFMPEG_ROOT} ]]; then
packaging/ffmpeg/build.sh
fi
- save_cache:
{% raw %}
key: ffmpeg-macos-v0-{{ checksum ".cachekey" }}
{% endraw %}
paths:
- third_party/ffmpeg
- persist_to_workspace:
root: third_party
paths:
- ffmpeg
build_ffmpeg_windows:
<<: *binary_common
machine:
resource_class: windows.xlarge
image: windows-server-2019-vs2019:stable
# Note:
# Unlike other Windows job, this job uses cmd.exe as shell because
# we need to invoke bash.exe from msys2 in ffmpeg build process, and doing so
# from different installation of bash.exe (the one from the VM) cause issue
shell: cmd.exe
steps:
- checkout
- run: date /t > .cachekey
- restore_cache:
{% raw %}
keys:
- ffmpeg-windows-{{ checksum ".cachekey" }}
{% endraw %}
- run: packaging\ffmpeg\build.bat
- save_cache:
{% raw %}
key: ffmpeg-windows-{{ checksum ".cachekey" }}
{% endraw %}
paths:
- third_party/ffmpeg
- persist_to_workspace:
root: third_party
paths:
- ffmpeg
workflows:
lint:
jobs:
- lint_python_and_config
unittest:
jobs:
- circleci_consistency
.circleci/regenerate.py deleted 100755 → 0
View file @ c7ac1aff
#!/usr/bin/env python3
"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.
Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.
Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.
See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
import os.path
import jinja2
import yaml
from jinja2 import select_autoescape
PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"]
def build_download_job(filter_branch):
job = {
"name": "download_third_parties",
}
if filter_branch:
job["filters"] = gen_filter_branch_tree(filter_branch)
return [{"download_third_parties": job}]
def build_ffmpeg_job(os_type, filter_branch):
job = {
"name": f"build_ffmpeg_{os_type}",
"requires": ["download_third_parties"],
}
if filter_branch:
job["filters"] = gen_filter_branch_tree(filter_branch)
job["python_version"] = "foo"
return [{f"build_ffmpeg_{os_type}": job}]
def gen_filter_branch_tree(*branches):
return {
"branches": {
"only": list(branches),
},
"tags": {
# Using a raw string here to avoid having to escape
# anything
"only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
},
}
def indent(indentation, data_list):
return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines())
def unittest_python_versions(os):
return {
"windows": PYTHON_VERSIONS[:1],
"macos": PYTHON_VERSIONS[:1],
"linux": PYTHON_VERSIONS,
}.get(os)
if __name__ == "__main__":
d = os.path.dirname(__file__)
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(d),
lstrip_blocks=True,
autoescape=select_autoescape(enabled_extensions=("html", "xml")),
)
with open(os.path.join(d, "config.yml"), "w") as f:
f.write(env.get_template("config.yml.in").render())
f.write("\n")
.circleci/smoke_test/docker/Dockerfile deleted 100644 → 0
View file @ c7ac1aff
# this Dockerfile is for torchaudio smoke test, it will be created periodically via CI system
# if you need to do it locally, follow below steps once you have Docker installed
# assuming you're within the directory where this Dockerfile located
# to test the build use : docker build . -t torchaudio/smoketest
# to upload the Dockerfile use build_and_push.sh script
FROM ubuntu:latest
RUN apt-get -qq update && apt-get -qq -y install curl bzip2 sox libsox-dev libsox-fmt-all \
&& curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
&& bash /tmp/miniconda.sh -bfp /usr/local \
&& rm -rf /tmp/miniconda.sh \
&& conda install -c conda-forge gcc \
&& conda install -y python=3 \
&& conda update conda \
&& apt-get -qq -y remove curl bzip2 \
&& apt-get -qq -y autoremove \
&& apt-get autoclean \
&& rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \
&& conda clean --all --yes
ENV PATH /opt/conda/bin:$PATH
RUN conda create -y --name python3.8 python=3.8
RUN conda create -y --name python3.9 python=3.9
RUN conda create -y --name python3.10 python=3.10
SHELL [ "/bin/bash", "-c" ]
RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc
RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.8 && conda install -y -c conda-forge sox && conda install -y numpy
RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.9 && conda install -y -c conda-forge sox && conda install -y numpy
RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.10 && conda install -y -c conda-forge sox && conda install -y numpy
CMD [ "/bin/bash"]
.circleci/smoke_test/docker/build_and_push.sh deleted 100755 → 0
View file @ c7ac1aff
#!/usr/bin/env bash
set -euo pipefail
datestr="$(date "+%Y%m%d")"
image="pytorch/torchaudio_unittest_base:smoke_test-${datestr}"
docker build -t "${image}" .
docker push "${image}"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment