make scipy an extra

README.md

@@ -451,6 +451,7 @@ Extras dependencies can be installed via `pip install -e ".[NAME]"`
 | openai        | For using OpenAI's models             |
 | optimum       | For running Intel OpenVINO models     |
 | promptsource  | For using PromptSource prompts        |
+| scipy         | Used in DROP postprocessing and in scripts/model_comparator.py |
 | sentencepiece | For using the sentencepiece tokenizer |
 | sparseml      | For using NM's SparseML models        |
 | testing       | For running library test suite        |

lm_eval/tasks/drop/utils.py

@@ -2,7 +2,15 @@ import re
 import string
 
 import numpy as np
-from scipy.optimize import linear_sum_assignment
+
+
+try:
+    from scipy.optimize import linear_sum_assignment
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        "`scipy` is required for DROP post-processing. \
+please install scipy via pip install lm-eval[scipy] or pip install -e .[scipy]",
+    )
 
 
 _ARTICLES = re.compile(r"\b(a|an|the)\b", re.UNICODE)

lm_eval/tasks/minerva_math/utils.py

@@ -12,7 +12,7 @@ try:
     from sympy.parsing.latex import parse_latex
 except ModuleNotFoundError:
     raise ModuleNotFoundError(
-        "`sympy` is required for generating translation task prompt templates. \
+        "`sympy` is required for checking equivalence of minerva_math tasks' answers thoroughly. \
 please install sympy via pip install lm-eval[math] or pip install -e .[math]",
     )
 

pyproject.toml

@@ -70,6 +70,7 @@ multilingual = ["nagisa>=0.2.7", "jieba>=0.42.1", "pycountry"]
 openai = ["openai==1.3.9", "tiktoken"]
 optimum = ["optimum[openvino]"]
 promptsource = ["promptsource>=0.2.3"]
+scipy = ["scipy"]
 sentencepiece = ["sentencepiece>=0.1.98"]
 sparseml = ["sparseml-nightly[llm]>=1.8.0.20240404"]
 testing = ["pytest", "pytest-cov", "pytest-xdist"]
@@ -89,6 +90,7 @@ all = [
     "lm_eval[multilingual]",
     "lm_eval[openai]",
     "lm_eval[promptsource]",
+    "lm_eval[scipy]",
     "lm_eval[sentencepiece]",
     "lm_eval[sparseml]",
     "lm_eval[testing]",

scripts/model_comparator.py

@@ -12,6 +12,15 @@ import lm_eval.models.utils
 from lm_eval import tasks, utils
 
 
+try:
+    import scipy.stats
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        "`scipy` is required for computing z-scores in hf-vllm comparisons via this script. \
+please install scipy via pip install lm-eval[scipy] or pip install -e .[scipy]",
+    )
+
+
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 eval_logger = utils.eval_logger