assistant prefill (#2615)

* add assistant prefix

* add arc_challenge from llama

* nit

* nit

* nit

* add assistant prefix

* add mmlu_llama

* nit

* nit

* Revert "nit"

This reverts commit 6a97f8356237305e375212b966b30e8de59dd4bc.

* fix regex bug

* add assistant_prefix to vllm

* add `Question:`

* add mmlu_pro

* add fewshot assistant_prefix

* use `assistant_prefill`

* typehints

* nits

* nits

* add to docs

* add readme

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_geography.yaml

+"dataset_name": "high_school_geography"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_social_sciences_tasks"
+"task": "mmlu_llama_high_school_geography"
+"task_alias": "high school geography"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_government_and_politics.yaml

+"dataset_name": "high_school_government_and_politics"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_social_sciences_tasks"
+"task": "mmlu_llama_high_school_government_and_politics"
+"task_alias": "high school government and politics"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_macroeconomics.yaml

+"dataset_name": "high_school_macroeconomics"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_social_sciences_tasks"
+"task": "mmlu_llama_high_school_macroeconomics"
+"task_alias": "high school macroeconomics"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_mathematics.yaml

+"dataset_name": "high_school_mathematics"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_stem_tasks"
+"task": "mmlu_llama_high_school_mathematics"
+"task_alias": "high school mathematics"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_microeconomics.yaml

+"dataset_name": "high_school_microeconomics"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_social_sciences_tasks"
+"task": "mmlu_llama_high_school_microeconomics"
+"task_alias": "high school microeconomics"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_physics.yaml

+"dataset_name": "high_school_physics"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_stem_tasks"
+"task": "mmlu_llama_high_school_physics"
+"task_alias": "high school physics"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_psychology.yaml

+"dataset_name": "high_school_psychology"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_social_sciences_tasks"
+"task": "mmlu_llama_high_school_psychology"
+"task_alias": "high school psychology"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_statistics.yaml

+"dataset_name": "high_school_statistics"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_stem_tasks"
+"task": "mmlu_llama_high_school_statistics"
+"task_alias": "high school statistics"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_us_history.yaml

+"dataset_name": "high_school_us_history"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_humanities_tasks"
+"task": "mmlu_llama_high_school_us_history"
+"task_alias": "high school us history"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_high_school_world_history.yaml

+"dataset_name": "high_school_world_history"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_humanities_tasks"
+"task": "mmlu_llama_high_school_world_history"
+"task_alias": "high school world history"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_human_aging.yaml

+"dataset_name": "human_aging"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_other_tasks"
+"task": "mmlu_llama_human_aging"
+"task_alias": "human aging"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_human_sexuality.yaml

+"dataset_name": "human_sexuality"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_social_sciences_tasks"
+"task": "mmlu_llama_human_sexuality"
+"task_alias": "human sexuality"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_international_law.yaml

+"dataset_name": "international_law"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_humanities_tasks"
+"task": "mmlu_llama_international_law"
+"task_alias": "international law"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_jurisprudence.yaml

+"dataset_name": "jurisprudence"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_humanities_tasks"
+"task": "mmlu_llama_jurisprudence"
+"task_alias": "jurisprudence"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_logical_fallacies.yaml

+"dataset_name": "logical_fallacies"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_humanities_tasks"
+"task": "mmlu_llama_logical_fallacies"
+"task_alias": "logical fallacies"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_machine_learning.yaml

+"dataset_name": "machine_learning"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_stem_tasks"
+"task": "mmlu_llama_machine_learning"
+"task_alias": "machine learning"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_management.yaml

+"dataset_name": "management"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_other_tasks"
+"task": "mmlu_llama_management"
+"task_alias": "management"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_marketing.yaml

+"dataset_name": "marketing"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_other_tasks"
+"task": "mmlu_llama_marketing"
+"task_alias": "marketing"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_medical_genetics.yaml

+"dataset_name": "medical_genetics"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_other_tasks"
+"task": "mmlu_llama_medical_genetics"
+"task_alias": "medical genetics"

lm_eval/tasks/llama3/instruct/mmlu/mmlu_miscellaneous.yaml

+"dataset_name": "miscellaneous"
+"include": "_continuation_template_yaml"
+"tag": "mmlu_llama_other_tasks"
+"task": "mmlu_llama_miscellaneous"
+"task_alias": "miscellaneous"