Merge branch 'EleutherAI:main' into main

lm_eval/tasks/leaderboard/math/math_counting_and_prob.yaml

+include: _template_yaml
+dataset_name: counting_and_probability
+task: leaderboard_math_counting_and_prob_hard

lm_eval/tasks/leaderboard/math/math_geometry.yaml

+include: _template_yaml
+dataset_name: geometry
+task: leaderboard_math_geometry_hard

lm_eval/tasks/leaderboard/math/math_intermediate_algebra.yaml

+include: _template_yaml
+dataset_name: intermediate_algebra
+task: leaderboard_math_intermediate_algebra_hard

lm_eval/tasks/leaderboard/math/math_num_theory.yaml

+include: _template_yaml
+dataset_name: number_theory
+task: leaderboard_math_num_theory_hard

lm_eval/tasks/leaderboard/math/math_prealgebra.yaml

+include: _template_yaml
+dataset_name: prealgebra
+task: leaderboard_math_prealgebra_hard

lm_eval/tasks/leaderboard/math/math_precalculus.yaml

+include: _template_yaml
+dataset_name: precalculus
+task: leaderboard_math_precalculus_hard

lm_eval/tasks/leaderboard/math/utils.py

+import re
+import signal
+from typing import Dict, List, Optional
+
+import datasets
+
+from lm_eval.utils import eval_logger
+
+
+try:
+    import sympy
+    from sympy.parsing.latex import parse_latex
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        "`sympy` is required for generating translation task prompt templates. \
+please install sympy via pip install lm-eval[math] or pip install -e .[math]",
+    )
+
+
+# taken from
+# https://github.com/wellecks/lm-evaluation-harness/blob/master/lm_eval/tasks/minerva_math.py
+def doc_to_text(doc: dict) -> str:
+    return "Problem:" + "\n" + doc["problem"] + "\n\n" + "Solution:"
+
+
+def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _process_doc(doc: dict) -> dict:
+        out_doc = {
+            "problem": doc["problem"],
+            "solution": doc["solution"],
+            "answer": normalize_final_answer(
+                remove_boxed(last_boxed_only_string(doc["solution"]))
+            ),
+        }
+        if getattr(doc, "few_shot", None) is not None:
+            out_doc["few_shot"] = True
+        return out_doc
+
+    return dataset.map(_process_doc)
+
+
+def list_fewshot_samples() -> list[dict]:
+    return [
+        {
+            "problem": "Find the domain of the expression  $\\frac{\\sqrt{x-2}}{\\sqrt{5-x}}$.}",
+            "solution": "The expressions inside each square root must be non-negative. Therefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{[2,5)}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.",
+            "few_shot": "1",
+        },
+        {
+            "problem": "If $\\det \\mathbf{A} = 2$ and $\\det \\mathbf{B} = 12,$ then find $\\det (\\mathbf{A} \\mathbf{B}).$",
+            "solution": "We have that $\\det (\\mathbf{A} \\mathbf{B}) = (\\det \\mathbf{A})(\\det \\mathbf{B}) = (2)(12) = \\boxed{24}.$\nFinal Answer: The final answer is $24$. I hope it is correct.",
+            "few_shot": "1",
+        },
+        {
+            "problem": "Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?",
+            "solution": "If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight.  If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight.  Equating this to 480 pounds, we can solve for $n$:\n\\begin{align*}\n30n&=480\\\n\\Rightarrow\\qquad n&=480/30=\\boxed{16}\n\\end{align*}\nFinal Answer: The final answer is $16$. I hope it is correct.",
+            "few_shot": "1",
+        },
+        {
+            "problem": "If the system of equations\n\n\\begin{align*}\n6x-4y&=a,\\\n6y-9x &=b.\n\\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,\nfind $\\frac{a}{b},$ assuming $b$ is nonzero.",
+            "solution": "If we multiply the first equation by $-\\frac{3}{2}$, we obtain\n\n$$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have\n\n$$-\\frac{3}{2}a=b\\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$\nFinal Answer: The final answer is $-\\frac{2}{3}$. I hope it is correct.",
+            "few_shot": "1",
+        },
+    ]
+
+
+def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
+    candidates = results[0]
+
+    unnormalized_answer = get_unnormalized_answer(candidates)
+    answer = normalize_final_answer(unnormalized_answer)
+
+    if is_equiv(answer, doc["answer"]):
+        retval = 1
+    else:
+        retval = 0
+
+    results = {
+        "exact_match": retval,
+    }
+    return results
+
+
+def last_boxed_only_string(string: str) -> Optional[str]:
+    idx = string.rfind("\\boxed")
+    if "\\boxed " in string:
+        return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0]
+    if idx < 0:
+        idx = string.rfind("\\fbox")
+        if idx < 0:
+            return None
+
+    i = idx
+    right_brace_idx = None
+    num_left_braces_open = 0
+    while i < len(string):
+        if string[i] == "{":
+            num_left_braces_open += 1
+        if string[i] == "}":
+            num_left_braces_open -= 1
+            if num_left_braces_open == 0:
+                right_brace_idx = i
+                break
+        i += 1
+
+    if right_brace_idx is None:
+        retval = None
+    else:
+        retval = string[idx : right_brace_idx + 1]
+
+    return retval
+
+
+def remove_boxed(s: str) -> str:
+    if "\\boxed " in s:
+        left = "\\boxed "
+        assert s[: len(left)] == left
+        return s[len(left) :]
+
+    left = "\\boxed{"
+
+    assert s[: len(left)] == left
+    assert s[-1] == "}"
+
+    return s[len(left) : -1]
+
+
+class timeout:
+    def __init__(self, seconds=1, error_message="Timeout"):
+        self.seconds = seconds
+        self.error_message = error_message
+
+    def handle_timeout(self, signum, frame):
+        raise TimeoutError(self.error_message)
+
+    def __enter__(self):
+        signal.signal(signal.SIGALRM, self.handle_timeout)
+        signal.alarm(self.seconds)
+
+    def __exit__(self, type, value, traceback):
+        signal.alarm(0)
+
+
+def is_equiv(x1: str, x2: str) -> bool:
+    """
+    x1 and x2 are normalized latex string
+    """
+    try:
+        with timeout(seconds=5):
+            try:
+                parsed_x1 = parse_latex(x1)
+                parsed_x2 = parse_latex(x2)
+            except (
+                sympy.parsing.latex.errors.LaTeXParsingError,
+                sympy.SympifyError,
+                TypeError,
+            ):
+                eval_logger.debug(f"couldn't parse one of {x1} or {x2}")
+                return False
+
+            try:
+                diff = parsed_x1 - parsed_x2
+            except TypeError:
+                eval_logger.debug(f"couldn't subtract {x1} and {x2}")
+                return False
+
+            try:
+                if sympy.simplify(diff) == 0:
+                    return True
+                else:
+                    return False
+            except ValueError:
+                eval_logger.debug(
+                    f"Had some trouble simplifying when comparing {x1} and {x2}"
+                )
+    except TimeoutError:
+        eval_logger.debug(f"Timed out comparing {x1} and {x2}")
+        return False
+    except ImportError as e:
+        eval_logger.error(e)
+        raise
+    except Exception as e:
+        eval_logger.debug(f"Failed comparing {x1} and {x2} with {e}")
+        return False
+
+
+def get_unnormalized_answer(text: str) -> str:
+    INVALID_ANSWER = "[invalidanswer]"
+    end_seq = "I hope it is correct."
+    text += end_seq
+    match = re.search(
+        r"Final Answer: The final answer is(.*?). I hope it is correct.",
+        text,
+    )
+    if match:
+        return match.group(1).strip()
+    else:
+        return INVALID_ANSWER
+
+
+SUBSTITUTIONS = [
+    ("an ", ""),
+    ("a ", ""),
+    (".$", "$"),
+    ("\\$", ""),
+    (r"\ ", ""),
+    (" ", ""),
+    ("mbox", "text"),
+    (",\\text{and}", ","),
+    ("\\text{and}", ","),
+    ("\\text{m}", "\\text{}"),
+]
+REMOVED_EXPRESSIONS = [
+    "square",
+    "ways",
+    "integers",
+    "dollars",
+    "mph",
+    "inches",
+    "ft",
+    "hours",
+    "km",
+    "units",
+    "\\ldots",
+    "sue",
+    "points",
+    "feet",
+    "minutes",
+    "digits",
+    "cents",
+    "degrees",
+    "cm",
+    "gm",
+    "pounds",
+    "meters",
+    "meals",
+    "edges",
+    "students",
+    "childrentickets",
+    "multiples",
+    "\\text{s}",
+    "\\text{.}",
+    "\\text{\ns}",
+    "\\text{}^2",
+    "\\text{}^3",
+    "\\text{\n}",
+    "\\text{}",
+    r"\mathrm{th}",
+    r"^\circ",
+    r"^{\circ}",
+    r"\;",
+    r",\!",
+    "{,}",
+    '"',
+    "\\dots",
+]
+
+
+def normalize_final_answer(final_answer: str) -> str:
+    """
+    Normalize a final answer to a quantitative reasoning question.
+
+    Copied character for character from appendix D of Lewkowycz et al. (2022)
+    """
+    final_answer = final_answer.split("=")[-1]
+
+    for before, after in SUBSTITUTIONS:
+        final_answer = final_answer.replace(before, after)
+    for expr in REMOVED_EXPRESSIONS:
+        final_answer = final_answer.replace(expr, "")
+
+    # Extract answer that is in LaTeX math, is bold,
+    # is surrounded by a box, etc.
+    final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer)
+    final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer)
+    final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer)
+    final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer)
+    final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer)
+
+    # Normalize shorthand TeX:
+    #  \fracab -> \frac{a}{b}
+    #  \frac{abc}{bef} -> \frac{abc}{bef}
+    #  \fracabc -> \frac{a}{b}c
+    #  \sqrta -> \sqrt{a}
+    #  \sqrtab -> sqrt{a}b
+    final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer)
+    final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer)
+    final_answer = final_answer.replace("$", "")
+
+    # Normalize 100,000 -> 100000
+    if final_answer.replace(",", "").isdigit():
+        final_answer = final_answer.replace(",", "")
+
+    return final_answer

lm_eval/tasks/leaderboard/mmlu_pro/mmlu_pro.yaml

+dataset_path: TIGER-Lab/MMLU-Pro # a copy of `cais/leaderboard_mmlu` with no auxiliary_train split
+task: leaderboard_mmlu_pro
+test_split: test
+fewshot_split: validation
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: !function utils.doc_to_text
+doc_to_choice: !function utils.doc_to_choice
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+num_fewshot: 5
+metadata:
+  version: 0.1

lm_eval/tasks/leaderboard/mmlu_pro/utils.py

+import string
+
+
+def doc_to_text(doc):
+    doc_to_text = f"{doc['question']}\n"
+
+    for i in range(len(doc["options"])):
+        doc_to_text += f"{string.ascii_uppercase[i]}. {doc['options'][i]}\n"
+
+    doc_to_text += "Answer:"
+    return doc_to_text
+
+
+def doc_to_choice(doc):
+    return [string.ascii_uppercase[i] for i in range(len(doc["options"]))]

lm_eval/tasks/leaderboard/musr/_musr.yaml

+group: leaderboard_musr
+task:
+  - leaderboard_musr_murder_mysteries
+  - leaderboard_musr_object_placements
+  - leaderboard_musr_team_allocation

lm_eval/tasks/leaderboard/musr/_template_yaml

+dataset_path: TAUR-Lab/MuSR
+output_type: multiple_choice
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "{{answer_choice}}"
+doc_to_choice: "{{choices}}"
+metric_list:
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/leaderboard/musr/musr_murder_mysteries.yaml

+include: "_template_yaml"
+task: leaderboard_musr_murder_mysteries
+test_split: murder_mysteries

lm_eval/tasks/leaderboard/musr/musr_object_placements.yaml

+include: "_template_yaml"
+task: leaderboard_musr_object_placements
+test_split: object_placements

lm_eval/tasks/leaderboard/musr/musr_team_allocation.yaml

+include: "_template_yaml"
+task: leaderboard_musr_team_allocation
+test_split: team_allocation

lm_eval/tasks/leaderboard/musr/utils.py

+import ast
+
+
+def doc_to_choice(doc):
+    """
+    Convert a doc to a choice.
+    """
+    return ast.literal_eval(doc["choices"])
+
+
+DOC_TO_TEXT = "{narrative}\n\n" "{question}\n\n" "{choices}\n" "Answer:"
+
+
+def doc_to_text(doc):
+    """
+    Convert a doc to text.
+    """
+    choices = ""
+    for i, choice in enumerate(ast.literal_eval(doc["choices"])):
+        choices += f"{i+1} - {choice}\n"
+
+    text = DOC_TO_TEXT.format(
+        narrative=doc["narrative"], question=doc["question"], choices=choices
+    )
+
+    return text

lm_eval/tasks/mathqa/mathqa.yaml

-group:
+tag:
   - math_word_problems
 task: mathqa
 dataset_path: math_qa

lm_eval/tasks/minerva_math/README.md

@@ -28,16 +28,11 @@ Eprint = {arXiv:2206.14858},
 }
 ```
 
-### Groups, Benchmarks and Tasks
-
-#### Benchmarks
-
-- `minerva_math`
+### Groups and Tasks
 
 #### Groups
 
-- `math_word_problems`
-- `generate_until`
+- `minerva_math`
 
 #### Tasks
 

lm_eval/tasks/minerva_math/minerva_math_algebra.yaml

@@ -9,7 +9,7 @@ training_split: train
 test_split: test
 doc_to_text:  !function utils.doc_to_text
 process_results: !function utils.process_results
-doc_to_target: "{{answer}}"
+doc_to_target: "{{answer if few_shot is undefined else solution}}"
 generation_kwargs:
   until:
     - "Problem:"
@@ -19,9 +19,11 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
-num_fewshot: 0
+num_fewshot: 4
 metadata:
   version: 1.0
-  num_fewshot: 4
 dataset_kwargs:
   trust_remote_code: true
+fewshot_config:
+  sampler: first_n
+  samples: !function utils.list_fewshot_samples

lm_eval/tasks/minerva_math/utils.py

@@ -20,49 +20,7 @@ please install sympy via pip install lm-eval[math] or pip install -e .[math]",
 # taken from
 # https://github.com/wellecks/lm-evaluation-harness/blob/master/lm_eval/tasks/minerva_math.py
 def doc_to_text(doc: dict) -> str:
-    PROMPT = r"""Problem:
-Find the domain of the expression  $\frac{\sqrt{x-2}}{\sqrt{5-x}}$.}
-
-Solution:
-The expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\boxed{[2,5)}$.
-Final Answer: The final answer is $[2,5)$. I hope it is correct.
-
-Problem:
-If $\det \mathbf{A} = 2$ and $\det \mathbf{B} = 12,$ then find $\det (\mathbf{A} \mathbf{B}).$
-
-Solution:
-We have that $\det (\mathbf{A} \mathbf{B}) = (\det \mathbf{A})(\det \mathbf{B}) = (2)(12) = \boxed{24}.$
-Final Answer: The final answer is $24$. I hope it is correct.
-
-Problem:
-Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?
-
-Solution:
-If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight.  If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight.  Equating this to 480 pounds, we can solve for $n$:
-\begin{align*}
-30n&=480\\
-\Rightarrow\qquad n&=480/30=\boxed{16}
-\end{align*}
-Final Answer: The final answer is $16$. I hope it is correct.
-
-Problem:
-If the system of equations
-
-\begin{align*}
-6x-4y&=a,\\
-6y-9x &=b.
-\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,
-find $\frac{a}{b},$ assuming $b$ is nonzero.
-
-Solution:
-If we multiply the first equation by $-\frac{3}{2}$, we obtain
-
-$$6y-9x=-\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have
-
-$$-\frac{3}{2}a=b\Rightarrow\frac{a}{b}=\boxed{-\frac{2}{3}}.$$
-Final Answer: The final answer is $-\frac{2}{3}$. I hope it is correct."""
-
-    return PROMPT + "\n\n" + "Problem:" + "\n" + doc["problem"] + "\n\n" + "Solution:"
+    return "Problem:" + "\n" + doc["problem"] + "\n\n" + "Solution:"
 
 
 def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
@@ -74,11 +32,38 @@ def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
                 remove_boxed(last_boxed_only_string(doc["solution"]))
             ),
         }
+        if getattr(doc, "few_shot", None) is not None:
+            out_doc["few_shot"] = True
         return out_doc
 
     return dataset.map(_process_doc)
 
 
+def list_fewshot_samples() -> list[dict]:
+    return [
+        {
+            "problem": "Find the domain of the expression  $\\frac{\\sqrt{x-2}}{\\sqrt{5-x}}$.}",
+            "solution": "The expressions inside each square root must be non-negative. Therefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{[2,5)}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.",
+            "few_shot": "1",
+        },
+        {
+            "problem": "If $\\det \\mathbf{A} = 2$ and $\\det \\mathbf{B} = 12,$ then find $\\det (\\mathbf{A} \\mathbf{B}).$",
+            "solution": "We have that $\\det (\\mathbf{A} \\mathbf{B}) = (\\det \\mathbf{A})(\\det \\mathbf{B}) = (2)(12) = \\boxed{24}.$\nFinal Answer: The final answer is $24$. I hope it is correct.",
+            "few_shot": "1",
+        },
+        {
+            "problem": "Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?",
+            "solution": "If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight.  If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight.  Equating this to 480 pounds, we can solve for $n$:\n\\begin{align*}\n30n&=480\\\n\\Rightarrow\\qquad n&=480/30=\\boxed{16}\n\\end{align*}\nFinal Answer: The final answer is $16$. I hope it is correct.",
+            "few_shot": "1",
+        },
+        {
+            "problem": "If the system of equations\n\n\\begin{align*}\n6x-4y&=a,\\\n6y-9x &=b.\n\\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,\nfind $\\frac{a}{b},$ assuming $b$ is nonzero.",
+            "solution": "If we multiply the first equation by $-\\frac{3}{2}$, we obtain\n\n$$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have\n\n$$-\\frac{3}{2}a=b\\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$\nFinal Answer: The final answer is $-\\frac{2}{3}$. I hope it is correct.",
+            "few_shot": "1",
+        },
+    ]
+
+
 def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
     candidates = results[0]
 

lm_eval/tasks/mmlu/_generate_configs.py

 """
 Take in a YAML, and output all "other" splits with this YAML
 """
+
 import argparse
 import logging
 import os
@@ -109,10 +110,9 @@ if __name__ == "__main__":
 
         yaml_dict = {
             "include": base_yaml_name,
-            "group": f"mmlu_{args.task_prefix}_{category}"
+            "tag": f"mmlu_{args.task_prefix}_{category}"
             if args.task_prefix != ""
             else f"mmlu_{category}",
-            "group_alias": category.replace("_", " "),
             "task": f"mmlu_{args.task_prefix}_{subject}"
             if args.task_prefix != ""
             else f"mmlu_{subject}",