assistant prefill (#2615)

* add assistant prefix

* add arc_challenge from llama

* nit

* nit

* nit

* add assistant prefix

* add mmlu_llama

* nit

* nit

* Revert "nit"

This reverts commit 6a97f8356237305e375212b966b30e8de59dd4bc.

* fix regex bug

* add assistant_prefix to vllm

* add `Question:`

* add mmlu_pro

* add fewshot assistant_prefix

* use `assistant_prefill`

* typehints

* nits

* nits

* add to docs

* add readme

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_computer_science.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_computer_science"
+task_alias: "computer_science"
+process_docs: !function utils.process_computer_science

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_economics.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_economics"
+task_alias: "economics"
+process_docs: !function utils.process_economics

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_engineering.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_engineering"
+task_alias: "engineering"
+process_docs: !function utils.process_engineering

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_health.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_health"
+task_alias: "health"
+process_docs: !function utils.process_health

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_history.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_history"
+task_alias: "history"
+process_docs: !function utils.process_history

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_law.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_law"
+task_alias: "law"
+process_docs: !function utils.process_law

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_math.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_math"
+task_alias: "math"
+process_docs: !function utils.process_math

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_other.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_other"
+task_alias: "other"
+process_docs: !function utils.process_other

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_philosophy.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_philosophy"
+task_alias: "philosophy"
+process_docs: !function utils.process_philosophy

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_physics.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_physics"
+task_alias: "physics"
+process_docs: !function utils.process_physics

lm_eval/tasks/llama3/instruct/mmlu_pro/mmlu_pro_psychology.yaml

+include: "_default_template_yaml"
+task: "mmlu_pro_llama_psychology"
+task_alias: "psychology"
+process_docs: !function utils.process_psychology

lm_eval/tasks/llama3/instruct/mmlu_pro/utils.py

+import re
+from functools import partial
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["category"] == subject)
+
+
+def fewshot_to_text(example):
+    text = example["cot_content"].removeprefix("A: Let's think step by step.").strip()
+    return re.sub(r"The answer is \(([A-Z])\)\.", r"The best answer is \1.", text)
+
+
+process_biology = partial(process_docs, subject="biology")
+process_business = partial(process_docs, subject="business")
+process_chemistry = partial(process_docs, subject="chemistry")
+process_computer_science = partial(process_docs, subject="computer science")
+process_economics = partial(process_docs, subject="economics")
+process_engineering = partial(process_docs, subject="engineering")
+process_health = partial(process_docs, subject="health")
+process_history = partial(process_docs, subject="history")
+process_law = partial(process_docs, subject="law")
+process_math = partial(process_docs, subject="math")
+process_other = partial(process_docs, subject="other")
+process_philosophy = partial(process_docs, subject="philosophy")
+process_physics = partial(process_docs, subject="physics")
+process_psychology = partial(process_docs, subject="psychology")