Skip to content

GitLab

Unverified Commit 80a0676a authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

CI dependency wheel caching (#6287) 

* Single workflow cache test




Remove cache dir, re-trigger cache


Only pip archives


Not sudo when pip

* All workflow cache

Remove no-cache-dir instruction


Remove last sudo occurrences


v0.3
parent 175cd45e
Hide whitespace changes
Inline Side-by-side
Showing with 83 additions and 19 deletions
.circleci/config.yml
View file @ 80a0676a
......@@ -10,9 +10,17 @@ jobs:
parallelism: 1
steps:
- checkout
- run: sudo pip install --upgrade pip
- run: sudo pip install .[sklearn,tf-cpu,torch,testing] --no-cache-dir
- run: sudo pip install codecov pytest-cov
- restore_cache:
keys:
- v0.3-torch_and_tf-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,tf-cpu,torch,testing]
- run: pip install codecov pytest-cov
- save_cache:
key: v0.3-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: python -m pytest -n 8 --dist=loadfile -s ./tests/ --cov | tee output.txt
- run: codecov
- store_artifacts:
......@@ -28,8 +36,16 @@ jobs:
parallelism: 1
steps:
- checkout
- run: sudo pip install --upgrade pip
- run: sudo pip install .[sklearn,torch,testing] --no-cache-dir
- restore_cache:
keys:
- v0.3-torch-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,torch,testing]
- save_cache:
key: v0.3-torch-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: python -m pytest -n 8 --dist=loadfile -s ./tests/ | tee output.txt
- store_artifacts:
path: ~/transformers/output.txt
......@@ -45,8 +61,16 @@ jobs:
parallelism: 1
steps:
- checkout
- run: sudo pip install --upgrade pip
- run: sudo pip install .[sklearn,tf-cpu,testing] --no-cache-dir
- restore_cache:
keys:
- v0.3-tf-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,tf-cpu,testing]
- save_cache:
key: v0.3-tf-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: python -m pytest -n 8 --dist=loadfile -s ./tests/ | tee output.txt
- store_artifacts:
path: ~/transformers/output.txt
......@@ -59,8 +83,16 @@ jobs:
RUN_CUSTOM_TOKENIZERS: yes
steps:
- checkout
- run: sudo pip install --upgrade pip
- run: sudo pip install .[ja,testing]
- restore_cache:
keys:
- v0.3-custom_tokenizers-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[ja,testing]
- save_cache:
key: v0.3-custom_tokenizers-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: python -m pytest -s ./tests/test_tokenization_bert_japanese.py | tee output.txt
- store_artifacts:
path: ~/transformers/output.txt
......@@ -75,9 +107,17 @@ jobs:
parallelism: 1
steps:
- checkout
- run: sudo pip install --upgrade pip
- run: sudo pip install .[sklearn,torch,testing] --no-cache-dir
- run: sudo pip install -r examples/requirements.txt
- restore_cache:
keys:
- v0.3-torch_examples-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,torch,testing]
- run: pip install -r examples/requirements.txt
- save_cache:
key: v0.3-torch_examples-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: python -m pytest -n 8 --dist=loadfile -rA -s ./examples/ | tee output.txt
- store_artifacts:
path: ~/transformers/output.txt
......@@ -88,8 +128,16 @@ jobs:
- image: circleci/python:3.6
steps:
- checkout
- run: sudo pip install --upgrade pip
- run: sudo pip install .[tf,torch,docs] --no-cache-dir
- restore_cache:
keys:
- v0.3-build_doc-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[tf,torch,docs]
- save_cache:
key: v0.3-build_doc-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: cd docs && make html SPHINXOPTS="-W"
- store_artifacts:
path: ./docs/_build
......@@ -102,7 +150,15 @@ jobs:
fingerprints:
- "5b:7a:95:18:07:8c:aa:76:4c:60:35:88:ad:60:56:71"
- checkout
- run: sudo pip install .[tf,torch,docs] --no-cache-dir
- restore_cache:
keys:
- v0.3-deploy_doc-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install .[tf,torch,docs]
- save_cache:
key: v0.3-deploy_doc-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: ./.circleci/deploy.sh
check_code_quality:
working_directory: ~/transformers
......@@ -112,10 +168,18 @@ jobs:
parallelism: 1
steps:
- checkout
- run: sudo pip install --upgrade pip
- restore_cache:
keys:
- v0.3-code_quality-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
# we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
- run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
- run: sudo pip install .[tf,torch,quality] --no-cache-dir
- run: pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
- run: pip install .[tf,torch,quality]
- save_cache:
key: v0.3-code_quality-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: black --check --line-length 119 --target-version py35 examples templates tests src utils
- run: isort --check-only --recursive examples templates tests src utils
- run: flake8 examples templates tests src utils
......@@ -127,7 +191,7 @@ jobs:
parallelism: 1
steps:
- checkout
- run: sudo pip install requests
- run: pip install requests
- run: python ./utils/link_tester.py
workflow_filters: &workflow_filters
filters:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment