Unverified Commit 54f0db40 authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

Add PT + TF automatic builds (#15860)



* Add PT + TF automatic builds

* Apply suggestions from code review
Co-authored-by: default avatarPatrick von Platen <patrick.v.platen@gmail.com>

* Wrap up
Co-authored-by: default avatarPatrick von Platen <patrick.v.platen@gmail.com>
parent 9863f7d2
name: Build docker images (scheduled) name: Build docker images (scheduled)
on: on:
push:
branches:
- docker-image*
repository_dispatch: repository_dispatch:
schedule: schedule:
- cron: "0 1 * * *" - cron: "0 1 * * *"
concurrency:
group: docker-images-builds
cancel-in-progress: false
jobs: jobs:
latest-docker: latest-docker:
name: "Latest PyTorch + TensorFlow [dev]" name: "Latest PyTorch + TensorFlow [dev]"
...@@ -34,6 +41,7 @@ jobs: ...@@ -34,6 +41,7 @@ jobs:
latest-torch-deepspeed-docker: latest-torch-deepspeed-docker:
name: "Latest PyTorch + DeepSpeed" name: "Latest PyTorch + DeepSpeed"
needs: latest-docker
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- -
...@@ -57,3 +65,57 @@ jobs: ...@@ -57,3 +65,57 @@ jobs:
REF=master REF=master
push: true push: true
tags: huggingface/transformers-pytorch-deepspeed-latest-gpu tags: huggingface/transformers-pytorch-deepspeed-latest-gpu
latest-pytorch:
name: "Latest PyTorch [dev]"
runs-on: ubuntu-latest
needs: latest-torch-deepspeed-docker
steps:
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Check out code
uses: actions/checkout@v2
-
name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
name: Build and push
uses: docker/build-push-action@v2
with:
context: ./docker/transformers-pytorch-gpu
build-args: |
REF=master
push: true
tags: huggingface/transformers-pytorch-gpu
latest-tensorflow:
needs: latest-pytorch
name: "Latest TensorFlow [dev]"
runs-on: ubuntu-latest
steps:
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Check out code
uses: actions/checkout@v2
-
name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
name: Build and push
uses: docker/build-push-action@v2
with:
context: ./docker/transformers-tensorflow-gpu
build-args: |
REF=master
push: true
tags: huggingface/transformers-tensorflow-gpu
...@@ -130,7 +130,7 @@ jobs: ...@@ -130,7 +130,7 @@ jobs:
machines: [multi-gpu-docker, single-gpu-docker] machines: [multi-gpu-docker, single-gpu-docker]
runs-on: ${{ matrix.machines }} runs-on: ${{ matrix.machines }}
container: container:
image: huggingface/transformers-pytorch-latest-gpu image: huggingface/transformers-pytorch-gpu
options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
needs: setup needs: setup
steps: steps:
...@@ -165,7 +165,7 @@ jobs: ...@@ -165,7 +165,7 @@ jobs:
machines: [multi-gpu-docker, single-gpu-docker] machines: [multi-gpu-docker, single-gpu-docker]
runs-on: ${{ matrix.machines }} runs-on: ${{ matrix.machines }}
container: container:
image: huggingface/transformers-tensorflow-latest-gpu image: huggingface/transformers-tensorflow-gpu
options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
needs: setup needs: setup
steps: steps:
......
...@@ -9,7 +9,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip ...@@ -9,7 +9,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip
ARG REF=master ARG REF=master
RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF
RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] RUN python3 -m pip install --no-cache-dir -e ./transformers[dev-torch,testing]
# If set to nothing, will install the latest version # If set to nothing, will install the latest version
ARG PYTORCH='' ARG PYTORCH=''
...@@ -19,6 +19,7 @@ RUN python3 -m pip uninstall -y tensorflow flax ...@@ -19,6 +19,7 @@ RUN python3 -m pip uninstall -y tensorflow flax
RUN python3 -m pip install --no-cache-dir torch-scatter -f https://data.pyg.org/whl/torch-$(python3 -c "from torch import version; print(version.__version__.split('+')[0])")+cpu.html RUN python3 -m pip install --no-cache-dir torch-scatter -f https://data.pyg.org/whl/torch-$(python3 -c "from torch import version; print(version.__version__.split('+')[0])")+cpu.html
RUN python3 -m pip install --no-cache-dir git+https://github.com/facebookresearch/detectron2.git pytesseract https://github.com/kpu/kenlm/archive/master.zip RUN python3 -m pip install --no-cache-dir git+https://github.com/facebookresearch/detectron2.git pytesseract https://github.com/kpu/kenlm/archive/master.zip
RUN python3 -m pip install -U "itsdangerous<2.1.0"
# When installing in editable mode, `transformers` is not recognized as a package. # When installing in editable mode, `transformers` is not recognized as a package.
# this line must be added in order for python to be aware of transformers. # this line must be added in order for python to be aware of transformers.
......
...@@ -9,13 +9,14 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip ...@@ -9,13 +9,14 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip
ARG REF=master ARG REF=master
RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF
RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] RUN python3 -m pip install --no-cache-dir -e ./transformers[dev-tensorflow,testing]
# If set to nothing, will install the latest version # If set to nothing, will install the latest version
ARG TENSORFLOW='' ARG TENSORFLOW=''
RUN [ ${#TENSORFLOW} -gt 0 ] && VERSION='tensorflow=='$TENSORFLOW'.*' || VERSION='tensorflow'; python3 -m pip install --no-cache-dir -U $VERSION RUN [ ${#TENSORFLOW} -gt 0 ] && VERSION='tensorflow=='$TENSORFLOW'.*' || VERSION='tensorflow'; python3 -m pip install --no-cache-dir -U $VERSION
RUN python3 -m pip uninstall -y torch flax RUN python3 -m pip uninstall -y torch flax
RUN python3 -m pip install -U "itsdangerous<2.1.0"
# When installing in editable mode, `transformers` is not recognized as a package. # When installing in editable mode, `transformers` is not recognized as a package.
# this line must be added in order for python to be aware of transformers. # this line must be added in order for python to be aware of transformers.
......
...@@ -309,6 +309,36 @@ extras["docs_specific"] = [] ...@@ -309,6 +309,36 @@ extras["docs_specific"] = []
# "docs" needs "all" to resolve all the references # "docs" needs "all" to resolve all the references
extras["docs"] = extras["all"] + extras["docs_specific"] extras["docs"] = extras["all"] + extras["docs_specific"]
extras["dev-torch"] = (
extras['testing']
+ extras['torch']
+ extras["sentencepiece"]
+ extras["tokenizers"]
+ extras["torch-speech"]
+ extras["vision"]
+ extras["integrations"]
+ extras["timm"]
+ extras["codecarbon"]
+ extras["quality"]
+ extras["ja"]
+ extras["docs_specific"]
+ extras["sklearn"]
+ extras["modelcreation"]
+ extras["onnxruntime"]
)
extras["dev-tensorflow"] = (
extras['testing']
+ extras['tf']
+ extras["sentencepiece"]
+ extras["tokenizers"]
+ extras["vision"]
+ extras["quality"]
+ extras["docs_specific"]
+ extras["sklearn"]
+ extras["modelcreation"]
+ extras["onnx"]
+ extras["tf-speech"]
)
extras["dev"] = ( extras["dev"] = (
extras["all"] extras["all"]
+ extras["testing"] + extras["testing"]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment