Skip to content

GitLab

Unverified Commit 1b6c8d48 authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

Update CI cache (#8126)

parent 378142af
Hide whitespace changes
Inline Side-by-side
Showing with 46 additions and 46 deletions
.circleci/config.yml
View file @ 1b6c8d48
...@@ -74,13 +74,13 @@ jobs: ...@@ -74,13 +74,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-torch_and_tf-{{ checksum "setup.py" }} - v0.4-torch_and_tf-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install git+https://github.com/huggingface/datasets - run: pip install git+https://github.com/huggingface/datasets
- run: pip install .[sklearn,tf-cpu,torch,testing] - run: pip install .[sklearn,tf-cpu,torch,testing]
- save_cache: - save_cache:
key: v0.3-{{ checksum "setup.py" }} key: v0.4-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pt_tf_cross_test --durations=0 | tee output.txt - run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pt_tf_cross_test --durations=0 | tee output.txt
...@@ -99,13 +99,13 @@ jobs: ...@@ -99,13 +99,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-torch-{{ checksum "setup.py" }} - v0.4-torch-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install git+https://github.com/huggingface/datasets - run: pip install git+https://github.com/huggingface/datasets
- run: pip install .[sklearn,torch,testing] - run: pip install .[sklearn,torch,testing]
- save_cache: - save_cache:
key: v0.3-torch-{{ checksum "setup.py" }} key: v0.4-torch-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: python -m pytest -n 8 --dist=loadfile -s --make_reports=tests ./tests/ | tee tests_output.txt - run: python -m pytest -n 8 --dist=loadfile -s --make_reports=tests ./tests/ | tee tests_output.txt
...@@ -113,7 +113,7 @@ jobs: ...@@ -113,7 +113,7 @@ jobs:
path: ~/transformers/tests_output.txt path: ~/transformers/tests_output.txt
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_tf: run_tests_tf:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
...@@ -126,13 +126,13 @@ jobs: ...@@ -126,13 +126,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-tf-{{ checksum "setup.py" }} - v0.4-tf-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install git+https://github.com/huggingface/datasets - run: pip install git+https://github.com/huggingface/datasets
- run: pip install .[sklearn,tf-cpu,testing] - run: pip install .[sklearn,tf-cpu,testing]
- save_cache: - save_cache:
key: v0.3-tf-{{ checksum "setup.py" }} key: v0.4-tf-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
...@@ -151,13 +151,13 @@ jobs: ...@@ -151,13 +151,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-flax-{{ checksum "setup.py" }} - v0.4-flax-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install git+https://github.com/huggingface/datasets - run: pip install git+https://github.com/huggingface/datasets
- run: sudo pip install .[flax,sklearn,torch,testing] - run: sudo pip install .[flax,sklearn,torch,testing]
- save_cache: - save_cache:
key: v0.3-flax-{{ checksum "setup.py" }} key: v0.4-flax-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
...@@ -176,13 +176,13 @@ jobs: ...@@ -176,13 +176,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-torch-{{ checksum "setup.py" }} - v0.4-torch-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install git+https://github.com/huggingface/datasets - run: pip install git+https://github.com/huggingface/datasets
- run: pip install .[sklearn,torch,testing] - run: pip install .[sklearn,torch,testing]
- save_cache: - save_cache:
key: v0.3-torch-{{ checksum "setup.py" }} key: v0.4-torch-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt
...@@ -201,13 +201,13 @@ jobs: ...@@ -201,13 +201,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-tf-{{ checksum "setup.py" }} - v0.4-tf-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install git+https://github.com/huggingface/datasets - run: pip install git+https://github.com/huggingface/datasets
- run: pip install .[sklearn,tf-cpu,testing] - run: pip install .[sklearn,tf-cpu,testing]
- save_cache: - save_cache:
key: v0.3-tf-{{ checksum "setup.py" }} key: v0.4-tf-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt
...@@ -224,13 +224,13 @@ jobs: ...@@ -224,13 +224,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-custom_tokenizers-{{ checksum "setup.py" }} - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install .[ja,testing] - run: pip install .[ja,testing]
- run: python -m unidic download - run: python -m unidic download
- save_cache: - save_cache:
key: v0.3-custom_tokenizers-{{ checksum "setup.py" }} key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: python -m pytest -s ./tests/test_tokenization_bert_japanese.py | tee output.txt - run: python -m pytest -s ./tests/test_tokenization_bert_japanese.py | tee output.txt
...@@ -249,13 +249,13 @@ jobs: ...@@ -249,13 +249,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-torch_examples-{{ checksum "setup.py" }} - v0.4-torch_examples-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install .[sklearn,torch,testing] - run: pip install .[sklearn,torch,testing]
- run: pip install -r examples/requirements.txt - run: pip install -r examples/requirements.txt
- save_cache: - save_cache:
key: v0.3-torch_examples-{{ checksum "setup.py" }} key: v0.4-torch_examples-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: python -m pytest -n 8 --dist=loadfile -s --make_reports=examples ./examples/ | tee examples_output.txt - run: python -m pytest -n 8 --dist=loadfile -s --make_reports=examples ./examples/ | tee examples_output.txt
...@@ -271,12 +271,12 @@ jobs: ...@@ -271,12 +271,12 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-build_doc-{{ checksum "setup.py" }} - v0.4-build_doc-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install .[tf,torch,sentencepiece,docs] - run: pip install .[tf,torch,sentencepiece,docs]
- save_cache: - save_cache:
key: v0.3-build_doc-{{ checksum "setup.py" }} key: v0.4-build_doc-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: cd docs && make html SPHINXOPTS="-W" - run: cd docs && make html SPHINXOPTS="-W"
...@@ -293,11 +293,11 @@ jobs: ...@@ -293,11 +293,11 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-deploy_doc-{{ checksum "setup.py" }} - v0.4-deploy_doc-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install .[tf,torch,sentencepiece,docs] - run: pip install .[tf,torch,sentencepiece,docs]
- save_cache: - save_cache:
key: v0.3-deploy_doc-{{ checksum "setup.py" }} key: v0.4-deploy_doc-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: ./.circleci/deploy.sh - run: ./.circleci/deploy.sh
...@@ -311,13 +311,13 @@ jobs: ...@@ -311,13 +311,13 @@ jobs:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- v0.3-code_quality-{{ checksum "setup.py" }} - v0.4-code_quality-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install isort - run: pip install isort
- run: pip install .[tf,torch,flax,quality] - run: pip install .[tf,torch,flax,quality]
- save_cache: - save_cache:
key: v0.3-code_quality-{{ checksum "setup.py" }} key: v0.4-code_quality-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- run: black --check examples templates tests src utils - run: black --check examples templates tests src utils
......
.github/workflows/self-push.yml
View file @ 1b6c8d48
name: Self-hosted runner (push) name: Self-hosted runner (push)
on: on:
push: push:
branches: branches:
- master - master
paths: paths:
- "src/**" - "src/**"
- "tests/**" - "tests/**"
- ".github/**" - ".github/**"
...@@ -22,7 +22,7 @@ jobs: ...@@ -22,7 +22,7 @@ jobs:
which python which python
python --version python --version
pip --version pip --version
- name: Current dir - name: Current dir
run: pwd run: pwd
- run: nvidia-smi - run: nvidia-smi
...@@ -32,7 +32,7 @@ jobs: ...@@ -32,7 +32,7 @@ jobs:
id: cache id: cache
with: with:
path: .env path: .env
key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }} key: v1-tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
- name: Create new python env (on self-hosted runners we have to handle isolation ourselves) - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
run: | run: |
...@@ -41,7 +41,7 @@ jobs: ...@@ -41,7 +41,7 @@ jobs:
which python which python
python --version python --version
pip --version pip --version
- name: Install dependencies - name: Install dependencies
run: | run: |
source .env/bin/activate source .env/bin/activate
...@@ -84,7 +84,7 @@ jobs: ...@@ -84,7 +84,7 @@ jobs:
id: cache id: cache
with: with:
path: .env path: .env
key: v0-tests_tf_torch_multiple_gpu-${{ hashFiles('setup.py') }} key: v1-tests_tf_torch_multiple_gpu-${{ hashFiles('setup.py') }}
- name: Create new python env (on self-hosted runners we have to handle isolation ourselves) - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
run: | run: |
......
.github/workflows/self-scheduled.yml
View file @ 1b6c8d48
...@@ -19,18 +19,18 @@ jobs: ...@@ -19,18 +19,18 @@ jobs:
id: cache id: cache
with: with:
path: .env path: .env
key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }} key: v1-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
- name: Python version - name: Python version
run: | run: |
which python which python
python --version python --version
pip --version pip --version
- name: Current dir - name: Current dir
run: pwd run: pwd
- run: nvidia-smi - run: nvidia-smi
- name: Create new python env (on self-hosted runners we have to handle isolation ourselves) - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
if: steps.cache.outputs.cache-hit != 'true' if: steps.cache.outputs.cache-hit != 'true'
run: | run: |
...@@ -39,7 +39,7 @@ jobs: ...@@ -39,7 +39,7 @@ jobs:
which python which python
python --version python --version
pip --version pip --version
- name: Install dependencies - name: Install dependencies
run: | run: |
source .env/bin/activate source .env/bin/activate
...@@ -81,7 +81,7 @@ jobs: ...@@ -81,7 +81,7 @@ jobs:
RUN_PIPELINE_TESTS: yes RUN_PIPELINE_TESTS: yes
run: | run: |
source .env/bin/activate source .env/bin/activate
python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_pipeline tests python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_pipeline tests
- name: test suite reports artifacts - name: test suite reports artifacts
if: ${{ always() }} if: ${{ always() }}
...@@ -101,7 +101,7 @@ jobs: ...@@ -101,7 +101,7 @@ jobs:
id: cache id: cache
with: with:
path: .env path: .env
key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }} key: v1-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }}
- name: Python version - name: Python version
run: | run: |
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment