Commit 3e1301bb authored by lintangsutawika's avatar lintangsutawika
Browse files

resolved merge conflict from latest version

parents fd9cd80f 070d31df
......@@ -20,49 +20,7 @@ please install sympy via pip install lm-eval[math] or pip install -e .[math]",
# taken from
# https://github.com/wellecks/lm-evaluation-harness/blob/master/lm_eval/tasks/minerva_math.py
def doc_to_text(doc: dict) -> str:
PROMPT = r"""Problem:
Find the domain of the expression $\frac{\sqrt{x-2}}{\sqrt{5-x}}$.}
Solution:
The expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\boxed{[2,5)}$.
Final Answer: The final answer is $[2,5)$. I hope it is correct.
Problem:
If $\det \mathbf{A} = 2$ and $\det \mathbf{B} = 12,$ then find $\det (\mathbf{A} \mathbf{B}).$
Solution:
We have that $\det (\mathbf{A} \mathbf{B}) = (\det \mathbf{A})(\det \mathbf{B}) = (2)(12) = \boxed{24}.$
Final Answer: The final answer is $24$. I hope it is correct.
Problem:
Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?
Solution:
If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$:
\begin{align*}
30n&=480\\
\Rightarrow\qquad n&=480/30=\boxed{16}
\end{align*}
Final Answer: The final answer is $16$. I hope it is correct.
Problem:
If the system of equations
\begin{align*}
6x-4y&=a,\\
6y-9x &=b.
\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,
find $\frac{a}{b},$ assuming $b$ is nonzero.
Solution:
If we multiply the first equation by $-\frac{3}{2}$, we obtain
$$6y-9x=-\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have
$$-\frac{3}{2}a=b\Rightarrow\frac{a}{b}=\boxed{-\frac{2}{3}}.$$
Final Answer: The final answer is $-\frac{2}{3}$. I hope it is correct."""
return PROMPT + "\n\n" + "Problem:" + "\n" + doc["problem"] + "\n\n" + "Solution:"
return "Problem:" + "\n" + doc["problem"] + "\n\n" + "Solution:"
def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
......@@ -74,11 +32,38 @@ def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
remove_boxed(last_boxed_only_string(doc["solution"]))
),
}
if getattr(doc, "few_shot", None) is not None:
out_doc["few_shot"] = True
return out_doc
return dataset.map(_process_doc)
def list_fewshot_samples() -> list[dict]:
return [
{
"problem": "Find the domain of the expression $\\frac{\\sqrt{x-2}}{\\sqrt{5-x}}$.}",
"solution": "The expressions inside each square root must be non-negative. Therefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{[2,5)}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.",
"few_shot": "1",
},
{
"problem": "If $\\det \\mathbf{A} = 2$ and $\\det \\mathbf{B} = 12,$ then find $\\det (\\mathbf{A} \\mathbf{B}).$",
"solution": "We have that $\\det (\\mathbf{A} \\mathbf{B}) = (\\det \\mathbf{A})(\\det \\mathbf{B}) = (2)(12) = \\boxed{24}.$\nFinal Answer: The final answer is $24$. I hope it is correct.",
"few_shot": "1",
},
{
"problem": "Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?",
"solution": "If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$:\n\\begin{align*}\n30n&=480\\\n\\Rightarrow\\qquad n&=480/30=\\boxed{16}\n\\end{align*}\nFinal Answer: The final answer is $16$. I hope it is correct.",
"few_shot": "1",
},
{
"problem": "If the system of equations\n\n\\begin{align*}\n6x-4y&=a,\\\n6y-9x &=b.\n\\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,\nfind $\\frac{a}{b},$ assuming $b$ is nonzero.",
"solution": "If we multiply the first equation by $-\\frac{3}{2}$, we obtain\n\n$$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have\n\n$$-\\frac{3}{2}a=b\\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$\nFinal Answer: The final answer is $-\\frac{2}{3}$. I hope it is correct.",
"few_shot": "1",
},
]
def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
candidates = results[0]
......
dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
output_type: multiple_choice
test_split: test
fewshot_split: dev
fewshot_config:
sampler: first_n
doc_to_text: "Question: {{question.strip()}}\nAnswer:"
doc_to_choice: "{{choices}}"
doc_to_target: "{{answer}}"
metadata:
version: 0.0
group: mmlu_continuation
task:
- mmlu_continuation_stem
- mmlu_continuation_other
- mmlu_continuation_social_sciences
- mmlu_continuation_humanities
"dataset_name": "abstract_algebra"
"description": "The following are questions (with answers) about abstract\
\ algebra.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_abstract_algebra"
"dataset_name": "anatomy"
"description": "The following are questions (with answers) about anatomy.\n\
\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_anatomy"
"dataset_name": "astronomy"
"description": "The following are questions (with answers) about astronomy.\n\
\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_astronomy"
"dataset_name": "business_ethics"
"description": "The following are questions (with answers) about business\
\ ethics.\n\n"
"group": "mmlu_continuation_other"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_business_ethics"
"dataset_name": "clinical_knowledge"
"description": "The following are questions (with answers) about clinical\
\ knowledge.\n\n"
"group": "mmlu_continuation_other"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_clinical_knowledge"
"dataset_name": "college_biology"
"description": "The following are questions (with answers) about college\
\ biology.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_college_biology"
"dataset_name": "college_chemistry"
"description": "The following are questions (with answers) about college\
\ chemistry.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_college_chemistry"
"dataset_name": "college_computer_science"
"description": "The following are questions (with answers) about college\
\ computer science.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_college_computer_science"
"dataset_name": "college_mathematics"
"description": "The following are questions (with answers) about college\
\ mathematics.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_college_mathematics"
"dataset_name": "college_medicine"
"description": "The following are questions (with answers) about college\
\ medicine.\n\n"
"group": "mmlu_continuation_other"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_college_medicine"
"dataset_name": "college_physics"
"description": "The following are questions (with answers) about college\
\ physics.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_college_physics"
"dataset_name": "computer_security"
"description": "The following are questions (with answers) about computer\
\ security.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_computer_security"
"dataset_name": "conceptual_physics"
"description": "The following are questions (with answers) about conceptual\
\ physics.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_conceptual_physics"
"dataset_name": "econometrics"
"description": "The following are questions (with answers) about econometrics.\n\
\n"
"group": "mmlu_continuation_social_sciences"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_econometrics"
"dataset_name": "electrical_engineering"
"description": "The following are questions (with answers) about electrical\
\ engineering.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_electrical_engineering"
"dataset_name": "elementary_mathematics"
"description": "The following are questions (with answers) about elementary\
\ mathematics.\n\n"
"group": "mmlu_continuation_stem"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_elementary_mathematics"
"dataset_name": "formal_logic"
"description": "The following are questions (with answers) about formal\
\ logic.\n\n"
"group": "mmlu_continuation_humanities"
"include": "_continuation_template_yaml"
"task": "mmlu_continuation_formal_logic"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment