Unpin `huggingface_hub` (#24667)

* fix

* fix

* fix

* [test all] commit

* [test all] commit

* [test all] commit

---------
Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

.circleci/config.yml

@@ -30,9 +30,9 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: pip install --upgrade pip
-            - run: pip install -U GitPython
-            - run: pip install -U .
+            - run: pip install --upgrade --upgrade-strategy eager pip
+            - run: pip install -U --upgrade-strategy eager GitPython
+            - run: pip install -U --upgrade-strategy eager .
             - run: mkdir -p test_preparation
             - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
             - store_artifacts:
@@ -104,9 +104,9 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: pip install --upgrade pip
-            - run: pip install -U GitPython
-            - run: pip install -U .
+            - run: pip install --upgrade --upgrade-strategy eager pip
+            - run: pip install -U --upgrade-strategy eager GitPython
+            - run: pip install -U --upgrade-strategy eager .
             - run: |
                   mkdir test_preparation
                   echo -n "tests" > test_preparation/test_list.txt
@@ -136,20 +136,20 @@ jobs:
             - checkout
             - restore_cache:
                   keys:
-                      - v0.6-code_quality-{{ checksum "setup.py" }}
-                      - v0.6-code-quality
+                      - v0.7-code_quality-{{ checksum "setup.py" }}
+                      - v0.7-code-quality
             - restore_cache:
                   keys:
-                      - v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
-                      - v0.6-code-quality-site-packages
-            - run: pip install --upgrade pip
-            - run: pip install -U .[all,quality]
+                      - v0.7-code_quality-{{ checksum "setup.py" }}-site-packages
+                      - v0.7-code-quality-site-packages
+            - run: pip install --upgrade --upgrade-strategy eager pip
+            - run: pip install -U --upgrade-strategy eager .[all,quality]
             - save_cache:
-                  key: v0.6-code_quality-{{ checksum "setup.py" }}
+                  key: v0.7-code_quality-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
             - save_cache:
-                  key: v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
+                  key: v0.7-code_quality-{{ checksum "setup.py" }}-site-packages
                   paths:
                       - '~/.pyenv/versions/'
             - run:
@@ -177,20 +177,20 @@ jobs:
             - checkout
             - restore_cache:
                   keys:
-                      - v0.6-repository_consistency-{{ checksum "setup.py" }}
-                      - v0.6-repository_consistency
+                      - v0.7-repository_consistency-{{ checksum "setup.py" }}
+                      - v0.7-repository_consistency
             - restore_cache:
                   keys:
-                      - v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
-                      - v0.6-repository_consistency-site-packages
-            - run: pip install --upgrade pip
-            - run: pip install -U .[all,quality]
+                      - v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages
+                      - v0.7-repository_consistency-site-packages
+            - run: pip install --upgrade --upgrade-strategy eager pip
+            - run: pip install -U --upgrade-strategy eager .[all,quality]
             - save_cache:
-                  key: v0.6-repository_consistency-{{ checksum "setup.py" }}
+                  key: v0.7-repository_consistency-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
             - save_cache:
-                  key: v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
+                  key: v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages
                   paths:
                       - '~/.pyenv/versions/'
             - run:

.circleci/create_circleci_config.py

@@ -52,7 +52,7 @@ class CircleCIJob:
     name: str
     additional_env: Dict[str, Any] = None
     cache_name: str = None
-    cache_version: str = "0.6"
+    cache_version: str = "0.7"
     docker_image: List[Dict[str, str]] = None
     install_steps: List[str] = None
     marker: Optional[str] = None
@@ -116,8 +116,6 @@ class CircleCIJob:
             },
         ]
         steps.extend([{"run": l} for l in self.install_steps])
-        # TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
-        steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@e4a419bf6bbaa95d14704cc781d3e81a49cef413"}})
         steps.append(
             {
                 "save_cache": {
@@ -244,10 +242,10 @@ torch_and_tf_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
         "git lfs install",
-        "pip install --upgrade pip",
-        "pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
-        "pip install -U tensorflow_probability",
-        "pip install -U git+https://github.com/huggingface/accelerate",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
+        "pip install -U --upgrade-strategy eager tensorflow_probability",
+        "pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
     ],
     marker="is_pt_tf_cross_test",
     pytest_options={"rA": None, "durations": 0},
@@ -259,9 +257,9 @@ torch_and_flax_job = CircleCIJob(
     additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
-        "pip install -U --upgrade pip",
-        "pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
-        "pip install -U git+https://github.com/huggingface/accelerate",
+        "pip install -U --upgrade-strategy eager --upgrade pip",
+        "pip install -U --upgrade-strategy eager .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
+        "pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
     ],
     marker="is_pt_flax_cross_test",
     pytest_options={"rA": None, "durations": 0},
@@ -272,9 +270,9 @@ torch_job = CircleCIJob(
     "torch",
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
-        "pip install --upgrade pip",
-        "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
-        "pip install -U git+https://github.com/huggingface/accelerate",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
+        "pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
     ],
     parallelism=1,
     pytest_num_workers=3,
@@ -285,9 +283,9 @@ tf_job = CircleCIJob(
     "tf",
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
-        "pip install --upgrade pip",
-        "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
-        "pip install -U tensorflow_probability",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
+        "pip install -U --upgrade-strategy eager tensorflow_probability",
     ],
     parallelism=1,
     pytest_num_workers=6,
@@ -299,8 +297,8 @@ flax_job = CircleCIJob(
     "flax",
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
-        "pip install --upgrade pip",
-        "pip install -U .[flax,testing,sentencepiece,flax-speech,vision]",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece,flax-speech,vision]",
     ],
     parallelism=1,
     pytest_options={"rA": None},
@@ -312,8 +310,8 @@ pipelines_torch_job = CircleCIJob(
     additional_env={"RUN_PIPELINE_TESTS": True},
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
-        "pip install --upgrade pip",
-        "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
     ],
     pytest_options={"rA": None},
     marker="is_pipeline_test",
@@ -325,9 +323,9 @@ pipelines_tf_job = CircleCIJob(
     additional_env={"RUN_PIPELINE_TESTS": True},
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y cmake",
-        "pip install --upgrade pip",
-        "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]",
-        "pip install -U tensorflow_probability",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,vision]",
+        "pip install -U --upgrade-strategy eager tensorflow_probability",
     ],
     pytest_options={"rA": None},
     marker="is_pipeline_test",
@@ -349,8 +347,8 @@ custom_tokenizers_job = CircleCIJob(
                 "sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n"
                 "sudo make install\n",
         },
-        "pip install --upgrade pip",
-        "pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
         "python -m unidic download",
     ],
     parallelism=None,
@@ -368,9 +366,9 @@ examples_torch_job = CircleCIJob(
     cache_name="torch_examples",
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
-        "pip install --upgrade pip",
-        "pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]",
-        "pip install -U -r examples/pytorch/_tests_requirements.txt",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[sklearn,torch,sentencepiece,testing,torch-speech]",
+        "pip install -U --upgrade-strategy eager -r examples/pytorch/_tests_requirements.txt",
     ],
 )
 
@@ -380,9 +378,9 @@ examples_tensorflow_job = CircleCIJob(
     cache_name="tensorflow_examples",
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y cmake",
-        "pip install --upgrade pip",
-        "pip install -U .[sklearn,tensorflow,sentencepiece,testing]",
-        "pip install -U -r examples/tensorflow/_tests_requirements.txt",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[sklearn,tensorflow,sentencepiece,testing]",
+        "pip install -U --upgrade-strategy eager -r examples/tensorflow/_tests_requirements.txt",
     ],
 )
 
@@ -391,9 +389,9 @@ examples_flax_job = CircleCIJob(
     "examples_flax",
     cache_name="flax_examples",
     install_steps=[
-        "pip install --upgrade pip",
-        "pip install -U .[flax,testing,sentencepiece]",
-        "pip install -U -r examples/flax/_tests_requirements.txt",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece]",
+        "pip install -U --upgrade-strategy eager -r examples/flax/_tests_requirements.txt",
     ],
 )
 
@@ -404,8 +402,8 @@ hub_job = CircleCIJob(
         "sudo apt-get -y update && sudo apt-get install git-lfs",
         'git config --global user.email "ci@dummy.com"',
         'git config --global user.name "ci"',
-        "pip install --upgrade pip",
-        "pip install -U .[torch,sentencepiece,testing]",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[torch,sentencepiece,testing]",
     ],
     marker="is_staging_test",
     pytest_num_workers=1,
@@ -416,8 +414,8 @@ onnx_job = CircleCIJob(
     "onnx",
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y cmake",
-        "pip install --upgrade pip",
-        "pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
     ],
     pytest_options={"k onnx": None},
     pytest_num_workers=1,
@@ -428,14 +426,16 @@ exotic_models_job = CircleCIJob(
     "exotic_models",
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
-        "pip install --upgrade pip",
-        "pip install -U .[torch,testing,vision]",
-        "pip install -U torchvision",
-        "pip install -U scipy",
-        "pip install -U 'git+https://github.com/facebookresearch/detectron2.git'",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[torch,testing,vision]",
+        "pip install -U --upgrade-strategy eager torchvision",
+        "pip install -U --upgrade-strategy eager scipy",
+        "pip install -U --upgrade-strategy eager 'git+https://github.com/facebookresearch/detectron2.git'",
         "sudo apt install tesseract-ocr",
-        "pip install -U pytesseract",
-        "pip install -U natten",
+        "pip install -U --upgrade-strategy eager pytesseract",
+        "pip install -U --upgrade-strategy eager natten",
+        # TODO (ydshieh): Remove this line once `https://github.com/facebookresearch/detectron2/issues/5010` is resolved
+        'pip install -U --upgrade-strategy eager "Pillow<10.0.0"',
     ],
     tests_to_run=[
         "tests/models/*layoutlmv*",
@@ -450,8 +450,8 @@ exotic_models_job = CircleCIJob(
 repo_utils_job = CircleCIJob(
     "repo_utils",
     install_steps=[
-        "pip install --upgrade pip",
-        "pip install -U .[quality,testing,torch]",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager .[quality,testing,torch]",
     ],
     parallelism=None,
     pytest_num_workers=1,
@@ -471,11 +471,11 @@ doc_test_job = CircleCIJob(
     additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"},
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
-        "pip install --upgrade pip",
-        "pip install -U -e .[dev]",
-        "pip install -U git+https://github.com/huggingface/accelerate",
-        "pip install --upgrade pytest pytest-sugar",
-        "pip install -U natten",
+        "pip install --upgrade --upgrade-strategy eager pip",
+        "pip install -U --upgrade-strategy eager -e .[dev]",
+        "pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
+        "pip install --upgrade --upgrade-strategy eager pytest pytest-sugar",
+        "pip install -U --upgrade-strategy eager natten",
         "find -name __pycache__ -delete",
         "find . -name \*.pyc -delete",
         # Add an empty file to keep the test step running correctly even no file is selected to be tested.