removed

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_computer_science.yaml

-"dataset_name": "computer_science"
-"description": "The following are multiple choice questions (with answers) about computer_science.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_stem"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_computer_science"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_economics.yaml

-"dataset_name": "economics"
-"description": "The following are multiple choice questions (with answers) about economics.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_social_sciences"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_economics"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_engineering.yaml

-"dataset_name": "engineering"
-"description": "The following are multiple choice questions (with answers) about engineering.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_stem"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_engineering"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_health.yaml

-"dataset_name": "health"
-"description": "The following are multiple choice questions (with answers) about health.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_other"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_health"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_history.yaml

-"dataset_name": "history"
-"description": "The following are multiple choice questions (with answers) about history.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_humanities"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_history"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_law.yaml

-"dataset_name": "law"
-"description": "The following are multiple choice questions (with answers) about law.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_humanities"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_law"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_math.yaml

-"dataset_name": "math"
-"description": "The following are multiple choice questions (with answers) about math.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_stem"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_math"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_other.yaml

-"dataset_name": "other"
-"description": "The following are multiple choice questions (with answers) about other.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_other"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_other"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_philosophy.yaml

-"dataset_name": "philosophy"
-"description": "The following are multiple choice questions (with answers) about philosophy.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_humanities"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_philosophy"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_physics.yaml

-"dataset_name": "physics"
-"description": "The following are multiple choice questions (with answers) about physics.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_stem"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_physics"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_psychology.yaml

-"dataset_name": "psychology"
-"description": "The following are multiple choice questions (with answers) about psychology.\n\
-  \n"
-"group": "mmlu_pro_flan_cot_zeroshot_social_sciences"
-"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
-"task": "mmlu_pro_flan_cot_zeroshot_psychology"

lm_eval/tasks/mmlu_pro/generative/_default_template_yaml

-dataset_path: sjyuxyz/MMLU-Pro-with-subset
-test_split: test
-fewshot_split: dev
-fewshot_config:
-  sampler: first_n
-output_type: generate_until
-doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nE. {{choices[4]}}\nF. {{choices[5]}}\nG. {{choices[6]}}\nH. {{choices[7]}}\nI. {{choices[8]}}\nJ. {{choices[9]}}\nAnswer:"
-doc_to_target: "{{['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'][answer]}}"
-generation_kwargs:
-  until:
-    - "</s>"
-    - "\n"
-metric_list:
-  - metric: exact_match
-    aggregation: mean
-    higher_is_better: true
-metadata:
-  version: 1.0

lm_eval/tasks/mmlu_pro/generative/_mmlu_pro.yaml

-group: mmlu_pro_generative
-task:
-  - mmlu_pro_stem_generative
-  - mmlu_pro_other_generative
-  - mmlu_pro_social_sciences_generative
-  - mmlu_pro_humanities_generative

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_biology.yaml

-"dataset_name": "biology"
-"description": "The following are multiple choice questions (with answers) about biology.\n\
-  \n"
-"group": "mmlu_pro_stem_generative"
-"group_alias": "stem"
-"include": "_default_template_yaml"
-"task": "mmlu_pro_biology_generative"
-"task_alias": "biology"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_business.yaml

-"dataset_name": "business"
-"description": "The following are multiple choice questions (with answers) about business.\n\
-  \n"
-"group": "mmlu_pro_other_generative"
-"group_alias": "other"
-"include": "_default_template_yaml"
-"task": "mmlu_pro_business_generative"
-"task_alias": "business"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_chemistry.yaml

-"dataset_name": "math"
-"description": "The following are multiple choice questions (with answers) about math.\n\
-  \n"
-"group": "mmlu_pro_stem_generative"
-"group_alias": "stem"
-"include": "_default_template_yaml"
-"task": "mmlu_pro_math_generative"
-"task_alias": "math"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_computer_science.yaml

-"dataset_name": "computer_science"
-"description": "The following are multiple choice questions (with answers) about computer_science.\n\
-  \n"
-"group": "mmlu_pro_stem_generative"
-"group_alias": "stem"
-"include": "_default_template_yaml"
-"task": "mmlu_pro_computer_science_generative"
-"task_alias": "computer_science"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_economics.yaml

-"dataset_name": "economics"
-"description": "The following are multiple choice questions (with answers) about economics.\n\
-  \n"
-"group": "mmlu_pro_social_sciences_generative"
-"group_alias": "social_sciences"
-"include": "_default_template_yaml"
-"task": "mmlu_pro_economics_generative"
-"task_alias": "economics"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_engineering.yaml

-"dataset_name": "engineering"
-"description": "The following are multiple choice questions (with answers) about engineering.\n\
-  \n"
-"group": "mmlu_pro_stem_generative"
-"group_alias": "stem"
-"include": "_default_template_yaml"
-"task": "mmlu_pro_engineering_generative"
-"task_alias": "engineering"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_health.yaml

-"dataset_name": "health"
-"description": "The following are multiple choice questions (with answers) about health.\n\
-  \n"
-"group": "mmlu_pro_other_generative"
-"group_alias": "other"
-"include": "_default_template_yaml"
-"task": "mmlu_pro_health_generative"
-"task_alias": "health"