adding hf_transfer (#1400)

* add hf_transfer

* update dependencies

* Delete stale `[linting]` extra

* Update README.md with extras table

---------
Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com>

README.md

@@ -314,6 +314,7 @@ Extras dependencies can be installed via `pip install -e ".[NAME]"`
 | anthropic     | For using Anthropic's models          |
 | dev           | For linting PRs and contributions     |
 | gptq          | For loading models with GPTQ          |
+| hf_transfer   | For speeding up HF Hub file downloads |
 | ifeval        | For running the IFEval task           |
 | neuronx       | For running on AWS inf2 instances     |
 | mamba         | For loading Mamba SSM models          |

lm_eval/models/__init__.py

@@ -9,3 +9,14 @@ from . import mamba_lm
 from . import optimum_lm
 from . import neuron_optimum
 # TODO: implement __all__
+
+
+import os
+
+try:
+    # enabling faster model download
+    import hf_transfer
+
+    os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+except ImportError:
+    pass

pyproject.toml

@@ -57,6 +57,7 @@ Repository = "https://github.com/EleutherAI/lm-evaluation-harness"
 anthropic = ["anthropic"]
 dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy"]
 gptq = ["auto-gptq[triton]>=0.6.0"]
+hf_transfer = ["hf_transfer"]
 ifeval = ["langdetect", "immutabledict"]
 neuronx = ["optimum[neuronx]"]
 mamba = ["mamba_ssm", "causal-conv1d==1.0.2"]
@@ -73,8 +74,8 @@ all = [
     "lm_eval[anthropic]",
     "lm_eval[dev]",
     "lm_eval[gptq]",
+    "lm_eval[hf_transfer]",
     "lm_eval[ifeval]",
-    "lm_eval[linting]",
     "lm_eval[mamba]",
     "lm_eval[math]",
     "lm_eval[multilingual]",