Commit 150234bd authored by Omkar Salpekar's avatar Omkar Salpekar Committed by Facebook GitHub Bot
Browse files

Cleaning up Deprecated Jobs from CCI Config (#3340)

Summary:
Cleaning up CCI configs that are no longer used.

Pull Request resolved: https://github.com/pytorch/audio/pull/3340

Reviewed By: mthrok

Differential Revision: D46077882

Pulled By: osalpekar

fbshipit-source-id: 0dce08fc14b5efc4517ab1f559e7ef7eb245af64
parent c0702338
...@@ -241,376 +241,6 @@ jobs: ...@@ -241,376 +241,6 @@ jobs:
paths: paths:
- ffmpeg - ffmpeg
binary_linux_wheel:
<<: *binary_common
docker:
- image: << parameters.wheel_docker_image >>
resource_class: 2xlarge+
steps:
- checkout
- attach_workspace:
at: third_party
- run:
command: |
export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
packaging/build_wheel.sh
environment:
USE_FFMPEG: true
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_linux_conda:
<<: *binary_common
docker:
- image: "<< parameters.conda_docker_image >>"
resource_class: 2xlarge+
steps:
- checkout
- load_conda_channel_flags
- attach_workspace:
at: third_party
- run:
name: Build conda packages
no_output_timeout: 30m
command: |
export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
packaging/build_conda.sh
environment:
USE_FFMPEG: true
- store_artifacts:
path: /opt/conda/conda-bld/linux-64
- persist_to_workspace:
root: /opt/conda
paths:
- "conda-bld/*"
binary_macos_wheel:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout
- load_conda_channel_flags
- attach_workspace:
at: third_party
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
# to build_wheel.sh
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
packaging/build_wheel.sh
environment:
USE_FFMPEG: true
USE_OPENMP: false
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_macos_conda:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout
- load_conda_channel_flags
- attach_workspace:
at: third_party
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
conda install -yq conda-build
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
packaging/build_conda.sh
environment:
USE_FFMPEG: true
USE_OPENMP: false
- store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
root: /Users/distiller/miniconda3
paths:
- "conda-bld/*"
binary_windows_wheel:
<<: *binary_common
executor:
name: windows-cpu
steps:
- checkout
- load_conda_channel_flags
- windows_install_cuda
- attach_workspace:
at: third_party
- run:
name: Build wheel packages
no_output_timeout: 30m
command: |
set -ex
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
bash packaging/build_wheel.sh
environment:
USE_FFMPEG: true
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_windows_conda:
<<: *binary_common
executor:
name: windows-cpu
steps:
- checkout
- load_conda_channel_flags
- windows_install_cuda
- attach_workspace:
at: third_party
- run:
name: Build conda packages
no_output_timeout: 30m
command: |
set -ex
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
bash packaging/build_conda.sh
environment:
USE_FFMPEG: true
- store_artifacts:
path: C:/tools/miniconda3/conda-bld/win-64
- persist_to_workspace:
root: C:/tools/miniconda3
paths:
- "conda-bld/*"
# Requires org-member context
binary_conda_upload:
docker:
- image: continuumio/miniconda
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
command: |
# Prevent credential from leaking
conda install -yq anaconda-client
set -x
anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
# Requires org-member context
binary_wheel_upload:
parameters:
subfolder:
description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
type: string
docker:
- image: cimg/python:3.8
steps:
- attach_workspace:
at: ~/workspace
- checkout
- designate_upload_channel
- run:
command: |
pip install --user awscli
export PATH="$HOME/.local/bin:$PATH"
# Prevent credential from leaking
set +x
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
smoke_test_linux_conda:
<<: *smoke_test_common
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly
conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
- checkout
- run:
name: smoke test
command: |
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install 'ffmpeg>=5.0,<6'
./test/smoke_test/run_smoke_test.sh
smoke_test_linux_conda_gpu:
<<: *smoke_test_common
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
- checkout
- run:
name: smoke test
command: |
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install 'ffmpeg>=5.0,<6'
./test/smoke_test/run_smoke_test.sh
smoke_test_linux_pip:
<<: *smoke_test_common
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html"
- checkout
- run:
name: smoke test
command: |
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOME}/workspace/ffmpeg/lib"
./test/smoke_test/run_smoke_test.sh
smoke_test_windows_conda:
<<: *binary_common
executor:
name: windows-cpu
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda update -y conda
conda env remove -n python${PYTHON_VERSION} || true
conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
conda activate python${PYTHON_VERSION}
conda install -v -y ${CONDA_CHANNEL_FLAGS:-} -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly
conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2)
- checkout
- run:
name: smoke test
command: |
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate python${PYTHON_VERSION}
conda install 'ffmpeg>=5.0,<6'
./test/smoke_test/run_smoke_test.sh
smoke_test_windows_conda_gpu:
<<: *binary_common
executor:
name: windows-gpu
steps:
- checkout
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- windows_install_cuda
- run:
name: Update CUDA driver
command: packaging/windows/internal/driver_update.bat
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda update -y conda
conda env remove -n python${PYTHON_VERSION} || true
conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
conda activate python${PYTHON_VERSION}
conda install -v -y ${CONDA_CHANNEL_FLAGS:-} -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch numpy 'ffmpeg>=5.0,<6' pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
# Install from torchaudio file
conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2)
- run:
name: smoke test
command: |
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate python${PYTHON_VERSION}
# Install sound backend
pip install PySoundFile
./test/smoke_test/run_smoke_test.sh
smoke_test_windows_pip:
<<: *binary_common
executor:
name: windows-cpu
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda update -y conda
conda env remove -n python${PYTHON_VERSION} || true
conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
conda activate python${PYTHON_VERSION}
pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html"
- checkout
- run:
name: smoke test
command: |
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate python${PYTHON_VERSION}
# Hack to load FFmpeg libraries
# Note: Depending on Python version, they search different paths.
# For 3.9, copying them in CWD works.
cp ~/workspace/ffmpeg/bin/* test/smoke_test/
# For 3.8 and 3.10, they must be in the same directory as the entrypoint lib
cp ~/workspace/ffmpeg/bin/* /C/tools/miniconda3/envs/python${PYTHON_VERSION}/lib/site-packages/torchaudio/lib/
./test/smoke_test/run_smoke_test.sh
unittest_linux_cpu: unittest_linux_cpu:
<<: *binary_common <<: *binary_common
docker: docker:
...@@ -786,18 +416,6 @@ jobs: ...@@ -786,18 +416,6 @@ jobs:
name: Run style check name: Run style check
command: .circleci/unittest/linux/scripts/run_style_checks.sh command: .circleci/unittest/linux/scripts/run_style_checks.sh
docstring_parameters_sync:
<<: *binary_common
docker:
- image: cimg/python:3.8
steps:
- checkout
- run:
name: Check parameters docstring sync
command: |
pip install --user pydocstyle
pydocstyle torchaudio
workflows: workflows:
lint: lint:
jobs: jobs:
......
...@@ -241,376 +241,6 @@ jobs: ...@@ -241,376 +241,6 @@ jobs:
paths: paths:
- ffmpeg - ffmpeg
binary_linux_wheel:
<<: *binary_common
docker:
- image: << parameters.wheel_docker_image >>
resource_class: 2xlarge+
steps:
- checkout
- attach_workspace:
at: third_party
- run:
command: |
export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
packaging/build_wheel.sh
environment:
USE_FFMPEG: true
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_linux_conda:
<<: *binary_common
docker:
- image: "<< parameters.conda_docker_image >>"
resource_class: 2xlarge+
steps:
- checkout
- load_conda_channel_flags
- attach_workspace:
at: third_party
- run:
name: Build conda packages
no_output_timeout: 30m
command: |
export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
packaging/build_conda.sh
environment:
USE_FFMPEG: true
- store_artifacts:
path: /opt/conda/conda-bld/linux-64
- persist_to_workspace:
root: /opt/conda
paths:
- "conda-bld/*"
binary_macos_wheel:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout
- load_conda_channel_flags
- attach_workspace:
at: third_party
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
# to build_wheel.sh
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
packaging/build_wheel.sh
environment:
USE_FFMPEG: true
USE_OPENMP: false
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_macos_conda:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout
- load_conda_channel_flags
- attach_workspace:
at: third_party
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
conda install -yq conda-build
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
packaging/build_conda.sh
environment:
USE_FFMPEG: true
USE_OPENMP: false
- store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
root: /Users/distiller/miniconda3
paths:
- "conda-bld/*"
binary_windows_wheel:
<<: *binary_common
executor:
name: windows-cpu
steps:
- checkout
- load_conda_channel_flags
- windows_install_cuda
- attach_workspace:
at: third_party
- run:
name: Build wheel packages
no_output_timeout: 30m
command: |
set -ex
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
bash packaging/build_wheel.sh
environment:
USE_FFMPEG: true
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_windows_conda:
<<: *binary_common
executor:
name: windows-cpu
steps:
- checkout
- load_conda_channel_flags
- windows_install_cuda
- attach_workspace:
at: third_party
- run:
name: Build conda packages
no_output_timeout: 30m
command: |
set -ex
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
bash packaging/build_conda.sh
environment:
USE_FFMPEG: true
- store_artifacts:
path: C:/tools/miniconda3/conda-bld/win-64
- persist_to_workspace:
root: C:/tools/miniconda3
paths:
- "conda-bld/*"
# Requires org-member context
binary_conda_upload:
docker:
- image: continuumio/miniconda
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
command: |
# Prevent credential from leaking
conda install -yq anaconda-client
set -x
anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
# Requires org-member context
binary_wheel_upload:
parameters:
subfolder:
description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
type: string
docker:
- image: cimg/python:3.8
steps:
- attach_workspace:
at: ~/workspace
- checkout
- designate_upload_channel
- run:
command: |
pip install --user awscli
export PATH="$HOME/.local/bin:$PATH"
# Prevent credential from leaking
set +x
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
smoke_test_linux_conda:
<<: *smoke_test_common
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly
conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
- checkout
- run:
name: smoke test
command: |
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install 'ffmpeg>=5.0,<6'
./test/smoke_test/run_smoke_test.sh
smoke_test_linux_conda_gpu:
<<: *smoke_test_common
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
- checkout
- run:
name: smoke test
command: |
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install 'ffmpeg>=5.0,<6'
./test/smoke_test/run_smoke_test.sh
smoke_test_linux_pip:
<<: *smoke_test_common
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html"
- checkout
- run:
name: smoke test
command: |
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOME}/workspace/ffmpeg/lib"
./test/smoke_test/run_smoke_test.sh
smoke_test_windows_conda:
<<: *binary_common
executor:
name: windows-cpu
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda update -y conda
conda env remove -n python${PYTHON_VERSION} || true
conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
conda activate python${PYTHON_VERSION}
conda install -v -y ${CONDA_CHANNEL_FLAGS:-} -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly
conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2)
- checkout
- run:
name: smoke test
command: |
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate python${PYTHON_VERSION}
conda install 'ffmpeg>=5.0,<6'
./test/smoke_test/run_smoke_test.sh
smoke_test_windows_conda_gpu:
<<: *binary_common
executor:
name: windows-gpu
steps:
- checkout
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- windows_install_cuda
- run:
name: Update CUDA driver
command: packaging/windows/internal/driver_update.bat
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda update -y conda
conda env remove -n python${PYTHON_VERSION} || true
conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
conda activate python${PYTHON_VERSION}
conda install -v -y ${CONDA_CHANNEL_FLAGS:-} -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch numpy 'ffmpeg>=5.0,<6' pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
# Install from torchaudio file
conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2)
- run:
name: smoke test
command: |
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate python${PYTHON_VERSION}
# Install sound backend
pip install PySoundFile
./test/smoke_test/run_smoke_test.sh
smoke_test_windows_pip:
<<: *binary_common
executor:
name: windows-cpu
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- load_conda_channel_flags
- run:
name: install binaries
no_output_timeout: 30m
command: |
set -x
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda update -y conda
conda env remove -n python${PYTHON_VERSION} || true
conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
conda activate python${PYTHON_VERSION}
pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html"
- checkout
- run:
name: smoke test
command: |
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate python${PYTHON_VERSION}
# Hack to load FFmpeg libraries
# Note: Depending on Python version, they search different paths.
# For 3.9, copying them in CWD works.
cp ~/workspace/ffmpeg/bin/* test/smoke_test/
# For 3.8 and 3.10, they must be in the same directory as the entrypoint lib
cp ~/workspace/ffmpeg/bin/* /C/tools/miniconda3/envs/python${PYTHON_VERSION}/lib/site-packages/torchaudio/lib/
./test/smoke_test/run_smoke_test.sh
unittest_linux_cpu: unittest_linux_cpu:
<<: *binary_common <<: *binary_common
docker: docker:
...@@ -786,18 +416,6 @@ jobs: ...@@ -786,18 +416,6 @@ jobs:
name: Run style check name: Run style check
command: .circleci/unittest/linux/scripts/run_style_checks.sh command: .circleci/unittest/linux/scripts/run_style_checks.sh
docstring_parameters_sync:
<<: *binary_common
docker:
- image: cimg/python:3.8
steps:
- checkout
- run:
name: Check parameters docstring sync
command: |
pip install --user pydocstyle
pydocstyle torchaudio
workflows: workflows:
lint: lint:
jobs: jobs:
......
...@@ -46,62 +46,6 @@ def build_ffmpeg_job(os_type, filter_branch): ...@@ -46,62 +46,6 @@ def build_ffmpeg_job(os_type, filter_branch):
return [{f"build_ffmpeg_{os_type}": job}] return [{f"build_ffmpeg_{os_type}": job}]
def build_workflow_pair(btype, os_type, python_version, cu_version, filter_branch, prefix="", upload=False):
w = []
base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}_{cu_version}"
w.append(generate_base_workflow(base_workflow_name, python_version, cu_version, filter_branch, os_type, btype))
if upload:
w.append(generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version))
if os_type != "macos":
pydistro = "pip" if btype == "wheel" else "conda"
w.append(
generate_smoketest_workflow(
pydistro, base_workflow_name, filter_branch, python_version, cu_version, os_type
)
)
return w
def docstring_parameters_sync_job(filter_branch):
job = {
"name": "docstring_parameters_sync",
"python_version": "3.8",
"requires": [
"binary_linux_wheel_py3.8_cpu",
],
}
if filter_branch:
job["filters"] = gen_filter_branch_tree(filter_branch)
return [{"docstring_parameters_sync": job}]
def generate_base_workflow(base_workflow_name, python_version, cu_version, filter_branch, os_type, btype):
d = {
"name": base_workflow_name,
"python_version": python_version,
"cuda_version": cu_version,
"requires": [f"build_ffmpeg_{os_type}"],
}
if btype == "conda":
d["conda_docker_image"] = f'pytorch/conda-builder:{cu_version.replace("cu1","cuda1")}'
elif cu_version.startswith("cu"):
d["wheel_docker_image"] = f'pytorch/manylinux-{cu_version.replace("cu1","cuda1")}'
elif cu_version.startswith("rocm"):
d["wheel_docker_image"] = f"pytorch/manylinux-rocm:{cu_version[len('rocm'):]}"
if filter_branch:
d["filters"] = gen_filter_branch_tree(filter_branch)
return {f"binary_{os_type}_{btype}": d}
def gen_filter_branch_tree(*branches): def gen_filter_branch_tree(*branches):
return { return {
"branches": { "branches": {
...@@ -115,41 +59,6 @@ def gen_filter_branch_tree(*branches): ...@@ -115,41 +59,6 @@ def gen_filter_branch_tree(*branches):
} }
def generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version):
d = {
"name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name),
"context": "org-member",
"requires": [base_workflow_name],
}
if btype == "wheel":
d["subfolder"] = "" if os_type == "macos" else cu_version + "/"
if filter_branch:
d["filters"] = gen_filter_branch_tree(filter_branch)
return {"binary_{btype}_upload".format(btype=btype): d}
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, cu_version, os_type):
smoke_suffix = f"smoke_test_{pydistro}".format(pydistro=pydistro)
d = {
"name": f"{base_workflow_name}_{smoke_suffix}",
"requires": [base_workflow_name],
"python_version": python_version,
"cuda_version": cu_version,
}
if filter_branch:
d["filters"] = gen_filter_branch_tree(filter_branch)
smoke_name = f"smoke_test_{os_type}_{pydistro}"
if pydistro == "conda" and (os_type == "linux" or os_type == "windows") and cu_version != "cpu":
smoke_name += "_gpu"
return {smoke_name: d}
def indent(indentation, data_list): def indent(indentation, data_list):
return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines()) return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines())
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment