Skip to content

GitLab

Unverified Commit 050ef145 authored by Yih-Dar's avatar Yih-Dar Committed by GitHub
Browse files

Unpin `huggingface_hub` (#24667) 



* fix

* fix

* fix

* [test all] commit

* [test all] commit

* [test all] commit

---------
Co-authored-by: default avatarydshieh <ydshieh@users.noreply.github.com>
parent bd9dfc23
Hide whitespace changes
Inline Side-by-side
Showing with 74 additions and 74 deletions
.circleci/config.yml
View file @ 050ef145
......@@ -30,9 +30,9 @@ jobs:
parallelism: 1
steps:
- checkout
- run: pip install --upgrade pip
- run: pip install -U GitPython
- run: pip install -U .
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager GitPython
- run: pip install -U --upgrade-strategy eager .
- run: mkdir -p test_preparation
- run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
- store_artifacts:
......@@ -104,9 +104,9 @@ jobs:
parallelism: 1
steps:
- checkout
- run: pip install --upgrade pip
- run: pip install -U GitPython
- run: pip install -U .
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager GitPython
- run: pip install -U --upgrade-strategy eager .
- run: |
mkdir test_preparation
echo -n "tests" > test_preparation/test_list.txt
......@@ -136,20 +136,20 @@ jobs:
- checkout
- restore_cache:
keys:
- v0.6-code_quality-{{ checksum "setup.py" }}
- v0.6-code-quality
- v0.7-code_quality-{{ checksum "setup.py" }}
- v0.7-code-quality
- restore_cache:
keys:
- v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
- v0.6-code-quality-site-packages
- run: pip install --upgrade pip
- run: pip install -U .[all,quality]
- v0.7-code_quality-{{ checksum "setup.py" }}-site-packages
- v0.7-code-quality-site-packages
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager .[all,quality]
- save_cache:
key: v0.6-code_quality-{{ checksum "setup.py" }}
key: v0.7-code_quality-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- save_cache:
key: v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
key: v0.7-code_quality-{{ checksum "setup.py" }}-site-packages
paths:
- '~/.pyenv/versions/'
- run:
......@@ -177,20 +177,20 @@ jobs:
- checkout
- restore_cache:
keys:
- v0.6-repository_consistency-{{ checksum "setup.py" }}
- v0.6-repository_consistency
- v0.7-repository_consistency-{{ checksum "setup.py" }}
- v0.7-repository_consistency
- restore_cache:
keys:
- v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
- v0.6-repository_consistency-site-packages
- run: pip install --upgrade pip
- run: pip install -U .[all,quality]
- v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages
- v0.7-repository_consistency-site-packages
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager .[all,quality]
- save_cache:
key: v0.6-repository_consistency-{{ checksum "setup.py" }}
key: v0.7-repository_consistency-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- save_cache:
key: v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
key: v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages
paths:
- '~/.pyenv/versions/'
- run:
......
.circleci/create_circleci_config.py
View file @ 050ef145
......@@ -52,7 +52,7 @@ class CircleCIJob:
name: str
additional_env: Dict[str, Any] = None
cache_name: str = None
cache_version: str = "0.6"
cache_version: str = "0.7"
docker_image: List[Dict[str, str]] = None
install_steps: List[str] = None
marker: Optional[str] = None
......@@ -116,8 +116,6 @@ class CircleCIJob:
},
]
steps.extend([{"run": l} for l in self.install_steps])
# TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@e4a419bf6bbaa95d14704cc781d3e81a49cef413"}})
steps.append(
{
"save_cache": {
......@@ -244,10 +242,10 @@ torch_and_tf_job = CircleCIJob(
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
"git lfs install",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U tensorflow_probability",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
marker="is_pt_tf_cross_test",
pytest_options={"rA": None, "durations": 0},
......@@ -259,9 +257,9 @@ torch_and_flax_job = CircleCIJob(
additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install -U --upgrade pip",
"pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install -U --upgrade-strategy eager --upgrade pip",
"pip install -U --upgrade-strategy eager .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
marker="is_pt_flax_cross_test",
pytest_options={"rA": None, "durations": 0},
......@@ -272,9 +270,9 @@ torch_job = CircleCIJob(
"torch",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
parallelism=1,
pytest_num_workers=3,
......@@ -285,9 +283,9 @@ tf_job = CircleCIJob(
"tf",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
"pip install -U tensorflow_probability",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
],
parallelism=1,
pytest_num_workers=6,
......@@ -299,8 +297,8 @@ flax_job = CircleCIJob(
"flax",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[flax,testing,sentencepiece,flax-speech,vision]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece,flax-speech,vision]",
],
parallelism=1,
pytest_options={"rA": None},
......@@ -312,8 +310,8 @@ pipelines_torch_job = CircleCIJob(
additional_env={"RUN_PIPELINE_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
],
pytest_options={"rA": None},
marker="is_pipeline_test",
......@@ -325,9 +323,9 @@ pipelines_tf_job = CircleCIJob(
additional_env={"RUN_PIPELINE_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]",
"pip install -U tensorflow_probability",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
],
pytest_options={"rA": None},
marker="is_pipeline_test",
......@@ -349,8 +347,8 @@ custom_tokenizers_job = CircleCIJob(
"sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n"
"sudo make install\n",
},
"pip install --upgrade pip",
"pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
"python -m unidic download",
],
parallelism=None,
......@@ -368,9 +366,9 @@ examples_torch_job = CircleCIJob(
cache_name="torch_examples",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]",
"pip install -U -r examples/pytorch/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,sentencepiece,testing,torch-speech]",
"pip install -U --upgrade-strategy eager -r examples/pytorch/_tests_requirements.txt",
],
)
......@@ -380,9 +378,9 @@ examples_tensorflow_job = CircleCIJob(
cache_name="tensorflow_examples",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tensorflow,sentencepiece,testing]",
"pip install -U -r examples/tensorflow/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tensorflow,sentencepiece,testing]",
"pip install -U --upgrade-strategy eager -r examples/tensorflow/_tests_requirements.txt",
],
)
......@@ -391,9 +389,9 @@ examples_flax_job = CircleCIJob(
"examples_flax",
cache_name="flax_examples",
install_steps=[
"pip install --upgrade pip",
"pip install -U .[flax,testing,sentencepiece]",
"pip install -U -r examples/flax/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece]",
"pip install -U --upgrade-strategy eager -r examples/flax/_tests_requirements.txt",
],
)
......@@ -404,8 +402,8 @@ hub_job = CircleCIJob(
"sudo apt-get -y update && sudo apt-get install git-lfs",
'git config --global user.email "ci@dummy.com"',
'git config --global user.name "ci"',
"pip install --upgrade pip",
"pip install -U .[torch,sentencepiece,testing]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,sentencepiece,testing]",
],
marker="is_staging_test",
pytest_num_workers=1,
......@@ -416,8 +414,8 @@ onnx_job = CircleCIJob(
"onnx",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
],
pytest_options={"k onnx": None},
pytest_num_workers=1,
......@@ -428,14 +426,16 @@ exotic_models_job = CircleCIJob(
"exotic_models",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
"pip install --upgrade pip",
"pip install -U .[torch,testing,vision]",
"pip install -U torchvision",
"pip install -U scipy",
"pip install -U 'git+https://github.com/facebookresearch/detectron2.git'",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,testing,vision]",
"pip install -U --upgrade-strategy eager torchvision",
"pip install -U --upgrade-strategy eager scipy",
"pip install -U --upgrade-strategy eager 'git+https://github.com/facebookresearch/detectron2.git'",
"sudo apt install tesseract-ocr",
"pip install -U pytesseract",
"pip install -U natten",
"pip install -U --upgrade-strategy eager pytesseract",
"pip install -U --upgrade-strategy eager natten",
# TODO (ydshieh): Remove this line once `https://github.com/facebookresearch/detectron2/issues/5010` is resolved
'pip install -U --upgrade-strategy eager "Pillow<10.0.0"',
],
tests_to_run=[
"tests/models/*layoutlmv*",
......@@ -450,8 +450,8 @@ exotic_models_job = CircleCIJob(
repo_utils_job = CircleCIJob(
"repo_utils",
install_steps=[
"pip install --upgrade pip",
"pip install -U .[quality,testing,torch]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[quality,testing,torch]",
],
parallelism=None,
pytest_num_workers=1,
......@@ -471,11 +471,11 @@ doc_test_job = CircleCIJob(
additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
"pip install --upgrade pip",
"pip install -U -e .[dev]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade pytest pytest-sugar",
"pip install -U natten",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager -e .[dev]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pytest pytest-sugar",
"pip install -U --upgrade-strategy eager natten",
"find -name __pycache__ -delete",
"find . -name \*.pyc -delete",
# Add an empty file to keep the test step running correctly even no file is selected to be tested.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment