Merge pull request #880 from EleutherAI/hotfixes

[Refactor] Hotfixes to big-refactor

lm_eval/api/task.py

@@ -632,6 +632,8 @@ class ConfigurableTask(Task):
         if self.fewshot_docs() is not None:
             self.sampler = samplers.get_sampler(
                 self.config.fewshot_config.get("sampler", "default")
+                if self.config.fewshot_config
+                else "default"
             )(list(self.fewshot_docs()), self, rnd=random.Random(1234))
 
         if self.has_test_docs():

lm_eval/tasks/minerva_math/utils.py

 import datasets
 import re
-import sympy
-from sympy.parsing.latex import parse_latex
 import signal
 from lm_eval.logger import eval_logger
 from typing import Optional, List, Dict
 
+try:
+    import sympy
+    from sympy.parsing.latex import parse_latex
+except ModuleNotFoundError:
+    raise Exception(
+        "`sympy` is required for generating translation task prompt templates. \
+please install sympy via pip install lm-eval[math] or pip install -e .[math]",
+    )
+
 
 # taken from
 # https://github.com/wellecks/lm-evaluation-harness/blob/master/lm_eval/tasks/minerva_math.py

lm_eval/utils.py

@@ -45,6 +45,14 @@ def escaped_split(text, sep_char, maxsplit=-1):
     return re.split(r"(?<!\\)" + sep_char, text, maxsplit)
 
 
+def handle_arg_string(arg):
+    if arg.lower() == "true":
+        return True
+    elif arg.lower() == "false":
+        return False
+    return arg
+
+
 def simple_parse_args_string(args_string):
     """
     Parses something like
@@ -55,7 +63,9 @@ def simple_parse_args_string(args_string):
     if not args_string:
         return {}
     arg_list = [arg for arg in args_string.split(",") if arg]
-    args_dict = {k: v for k, v in [arg.split("=") for arg in arg_list]}
+    args_dict = {
+        k: handle_arg_string(v) for k, v in [arg.split("=") for arg in arg_list]
+    }
     return args_dict