Skip to content

GitLab

Unverified Commit a5579189 authored by Andrey Talman's avatar Andrey Talman Committed by GitHub
Browse files

Cleanup circleci binary generation (#7591) 


Co-authored-by: default avatarPhilip Meier <github.pmeier@posteo.de>
parent 353b71b0
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 487 deletions
.circleci/config.yml
View file @ a5579189
...@@ -124,144 +124,6 @@ jobs: ...@@ -124,144 +124,6 @@ jobs:
python .circleci/regenerate.py python .circleci/regenerate.py
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
binary_win_conda:
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Build conda packages
no_output_timeout: 30m
command: |
set -ex
source packaging/windows/internal/vc_install_helper.sh
packaging/windows/internal/cuda_install.bat
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
packaging/build_conda.sh
rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
- store_artifacts:
path: C:/tools/miniconda3/conda-bld/win-64
- persist_to_workspace:
root: C:/tools/miniconda3/conda-bld/win-64
paths:
- "*"
- store_test_results:
path: build_results/
binary_win_wheel:
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Build wheel packages
no_output_timeout: 30m
command: |
set -ex
source packaging/windows/internal/vc_install_helper.sh
packaging/windows/internal/cuda_install.bat
packaging/build_wheel.sh
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
- store_test_results:
path: build_results/
binary_macos_wheel:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout_merge
- designate_upload_channel
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
# to build_wheel.sh
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
packaging/build_wheel.sh
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_macos_conda:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
conda install -yq conda-build
packaging/build_conda.sh
- store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
root: /Users/distiller/miniconda3/conda-bld/osx-64
paths:
- "*"
- store_test_results:
path: build_results/
# Requires org-member context
binary_conda_upload:
docker:
- image: continuumio/miniconda
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
command: |
# Prevent credential from leaking
conda install -yq anaconda-client
set -x
anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
# Requires org-member context
binary_wheel_upload:
parameters:
subfolder:
description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
type: string
docker:
- image: cimg/python:3.8
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- checkout
- pip_install:
args: awscli
- run:
command: |
export PATH="$HOME/.local/bin:$PATH"
# Prevent credential from leaking
set +x
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
smoke_test_docker_image_build: smoke_test_docker_image_build:
machine: machine:
image: ubuntu-2004:202104-01 image: ubuntu-2004:202104-01
...@@ -372,10 +234,6 @@ workflows: ...@@ -372,10 +234,6 @@ workflows:
jobs: jobs:
- circleci_consistency - circleci_consistency
build:
jobs:
[]
cmake: cmake:
jobs: jobs:
- cmake_linux_cpu: - cmake_linux_cpu:
...@@ -400,9 +258,6 @@ workflows: ...@@ -400,9 +258,6 @@ workflows:
name: cmake_macos_cpu name: cmake_macos_cpu
python_version: '3.8' python_version: '3.8'
nightly:
jobs:
[]
docker_build: docker_build:
triggers: triggers:
- schedule: - schedule:
......
.circleci/config.yml.in
View file @ a5579189
...@@ -124,144 +124,6 @@ jobs: ...@@ -124,144 +124,6 @@ jobs:
python .circleci/regenerate.py python .circleci/regenerate.py
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
binary_win_conda:
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Build conda packages
no_output_timeout: 30m
command: |
set -ex
source packaging/windows/internal/vc_install_helper.sh
packaging/windows/internal/cuda_install.bat
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
packaging/build_conda.sh
rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
- store_artifacts:
path: C:/tools/miniconda3/conda-bld/win-64
- persist_to_workspace:
root: C:/tools/miniconda3/conda-bld/win-64
paths:
- "*"
- store_test_results:
path: build_results/
binary_win_wheel:
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Build wheel packages
no_output_timeout: 30m
command: |
set -ex
source packaging/windows/internal/vc_install_helper.sh
packaging/windows/internal/cuda_install.bat
packaging/build_wheel.sh
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
- store_test_results:
path: build_results/
binary_macos_wheel:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout_merge
- designate_upload_channel
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
# to build_wheel.sh
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
packaging/build_wheel.sh
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_macos_conda:
<<: *binary_common
macos:
xcode: "14.0"
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
conda install -yq conda-build
packaging/build_conda.sh
- store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
root: /Users/distiller/miniconda3/conda-bld/osx-64
paths:
- "*"
- store_test_results:
path: build_results/
# Requires org-member context
binary_conda_upload:
docker:
- image: continuumio/miniconda
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
command: |
# Prevent credential from leaking
conda install -yq anaconda-client
set -x
anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
# Requires org-member context
binary_wheel_upload:
parameters:
subfolder:
description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
type: string
docker:
- image: cimg/python:3.8
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- checkout
- pip_install:
args: awscli
- run:
command: |
export PATH="$HOME/.local/bin:$PATH"
# Prevent credential from leaking
set +x
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
smoke_test_docker_image_build: smoke_test_docker_image_build:
machine: machine:
image: ubuntu-2004:202104-01 image: ubuntu-2004:202104-01
...@@ -372,17 +234,10 @@ workflows: ...@@ -372,17 +234,10 @@ workflows:
jobs: jobs:
- circleci_consistency - circleci_consistency
build:
jobs:
{{ build_workflows(windows_latest_only=True) }}
cmake: cmake:
jobs: jobs:
{{ cmake_workflows() }} {{ cmake_workflows() }}
nightly:
jobs:
{{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
docker_build: docker_build:
triggers: triggers:
- schedule: - schedule:
......
.circleci/regenerate.py
View file @ a5579189
...@@ -21,202 +21,6 @@ import yaml ...@@ -21,202 +21,6 @@ import yaml
from jinja2 import select_autoescape from jinja2 import select_autoescape
PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"]
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
def build_workflows(prefix="", filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
w = []
# Don't generate anything for build workflow
return indent(indentation, w)
for btype in ["wheel", "conda"]:
for os_type in ["linux", "macos", "win"]:
python_versions = PYTHON_VERSIONS
cu_versions_dict = {
"linux": ["cpu", "cu117", "cu118", "cu121", "rocm5.2", "rocm5.3"],
"win": ["cpu", "cu117", "cu118", "cu121"],
"macos": ["cpu"],
}
cu_versions = cu_versions_dict[os_type]
for python_version in python_versions:
for cu_version in cu_versions:
# ROCm conda packages not yet supported
if cu_version.startswith("rocm") and btype == "conda":
continue
for unicode in [False]:
fb = filter_branch
if (
windows_latest_only
and os_type == "win"
and filter_branch is None
and (
python_version != python_versions[-1]
or (cu_version not in [cu_versions[0], cu_versions[-1]])
)
):
fb = "main"
if not fb and (
os_type == "linux" and cu_version == "cpu" and btype == "wheel" and python_version == "3.8"
):
# the fields must match the build_docs "requires" dependency
fb = "/.*/"
# Disable all Linux Wheels Workflows from CircleCI
if os_type == "linux" and btype == "wheel":
continue
# Disable all Macos Wheels Workflows from CircleCI.
if os_type == "macos" and btype == "wheel":
continue
# Disable all non-Windows Conda workflows
if os_type != "win" and btype == "conda":
continue
w += workflow_pair(
btype, os_type, python_version, cu_version, unicode, prefix, upload, filter_branch=fb
)
return indent(indentation, w)
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix="", upload=False, *, filter_branch=None):
w = []
unicode_suffix = "u" if unicode else ""
base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"
w.append(
generate_base_workflow(
base_workflow_name, python_version, cu_version, unicode, os_type, btype, filter_branch=filter_branch
)
)
# For the remaining py3.8 Linux Wheels job left around for the docs build,
# we'll disable uploads.
if os_type == "linux" and btype == "wheel":
upload = False
if upload:
w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
# disable smoke tests, they are broken and needs to be fixed
# if filter_branch == "nightly" and os_type in ["linux", "win"]:
# pydistro = "pip" if btype == "wheel" else "conda"
# w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
return w
manylinux_images = {
"cu117": "pytorch/manylinux-cuda117",
"cu118": "pytorch/manylinux-cuda118",
"cu121": "pytorch/manylinux-cuda121",
}
def get_manylinux_image(cu_version):
if cu_version == "cpu":
return "pytorch/manylinux-cpu"
elif cu_version.startswith("cu"):
cu_suffix = cu_version[len("cu") :]
return f"pytorch/manylinux-cuda{cu_suffix}"
elif cu_version.startswith("rocm"):
rocm_suffix = cu_version[len("rocm") :]
return f"pytorch/manylinux-rocm:{rocm_suffix}"
def get_conda_image(cu_version):
if cu_version == "cpu":
return "pytorch/conda-builder:cpu"
elif cu_version.startswith("cu"):
cu_suffix = cu_version[len("cu") :]
return f"pytorch/conda-builder:cuda{cu_suffix}"
def generate_base_workflow(
base_workflow_name, python_version, cu_version, unicode, os_type, btype, *, filter_branch=None
):
d = {
"name": base_workflow_name,
"python_version": python_version,
"cu_version": cu_version,
}
if os_type != "win" and unicode:
d["unicode_abi"] = "1"
if os_type != "win":
d["wheel_docker_image"] = get_manylinux_image(cu_version)
# ROCm conda packages not yet supported
if "rocm" not in cu_version:
d["conda_docker_image"] = get_conda_image(cu_version)
if filter_branch is not None:
d["filters"] = {
"branches": {"only": filter_branch},
"tags": {
# Using a raw string here to avoid having to escape
# anything
"only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
},
}
w = f"binary_{os_type}_{btype}"
return {w: d}
def gen_filter_branch_tree(*branches, tags_list=None):
filter_dict = {"branches": {"only": [b for b in branches]}}
if tags_list is not None:
filter_dict["tags"] = {"only": tags_list}
return filter_dict
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
d = {
"name": f"{base_workflow_name}_upload",
"context": "org-member",
"requires": [base_workflow_name],
}
if btype == "wheel":
d["subfolder"] = "" if os_type == "macos" else cu_version + "/"
if filter_branch is not None:
d["filters"] = {
"branches": {"only": filter_branch},
"tags": {
# Using a raw string here to avoid having to escape
# anything
"only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
},
}
return {f"binary_{btype}_upload": d}
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):
required_build_suffix = "_upload"
required_build_name = base_workflow_name + required_build_suffix
smoke_suffix = f"smoke_test_{pydistro}"
d = {
"name": f"{base_workflow_name}_{smoke_suffix}",
"requires": [required_build_name],
"python_version": python_version,
}
if filter_branch:
d["filters"] = gen_filter_branch_tree(filter_branch)
return {f"smoke_test_{os_type}_{pydistro}": d}
def indent(indentation, data_list): def indent(indentation, data_list):
return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines()) return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines())
...@@ -249,7 +53,6 @@ if __name__ == "__main__": ...@@ -249,7 +53,6 @@ if __name__ == "__main__":
with open(os.path.join(d, "config.yml"), "w") as f: with open(os.path.join(d, "config.yml"), "w") as f:
f.write( f.write(
env.get_template("config.yml.in").render( env.get_template("config.yml.in").render(
build_workflows=build_workflows,
cmake_workflows=cmake_workflows, cmake_workflows=cmake_workflows,
) )
) )
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment