added generative mmlu-pro

lm_eval/tasks/mmlu_pro/default/mmlu_pro_history.yaml

-"dataset_name": "law"
-"description": "The following are multiple choice questions (with answers) about law.\n\
+"dataset_name": "history"
+"description": "The following are multiple choice questions (with answers) about history.\n\
   \n"
 "group": "mmlu_pro_humanities"
 "group_alias": "humanities"
 "include": "_default_template_yaml"
-"task": "mmlu_pro_law"
-"task_alias": "law"
+"task": "mmlu_pro_history"
+"task_alias": "history"

lm_eval/tasks/mmlu_pro/generative/_default_template_yaml

+dataset_path: sjyuxyz/MMLU-Pro-with-subset
+test_split: test
+fewshot_split: validation
+fewshot_config:
+  sampler: first_n
+output_type: generate_until
+doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nE. {{choices[4]}}\nF. {{choices[5]}}\nG. {{choices[6]}}\nH. {{choices[7]}}\nI. {{choices[8]}}\nJ. {{choices[9]}}\nAnswer:"
+doc_to_target: "{{['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'][answer]}}"
+generation_kwargs:
+  until:
+    - "</s>"
+    - "\n"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/mmlu_pro/generative/_mmlu_pro.yaml

+group: mmlu_pro_generative
+task:
+  - mmlu_pro_stem_generative
+  - mmlu_pro_other_generative
+  - mmlu_pro_social_sciences_generative
+  - mmlu_pro_humanities_generative

lm_eval/tasks/mmlu_pro/default/mmlu_abstract_algebra.yaml → lm_eval/tasks/mmlu_pro/generative/mmlu_pro_biology.yaml

-"dataset_name": "abstract_algebra"
-"description": "The following are multiple choice questions (with answers) about abstract\
-  \ algebra.\n\n"
-"group": "mmlu_stem"
+"dataset_name": "biology"
+"description": "The following are multiple choice questions (with answers) about biology.\n\
+  \n"
+"group": "mmlu_pro_stem_generative"
 "group_alias": "stem"
 "include": "_default_template_yaml"
-"task": "mmlu_abstract_algebra"
-"task_alias": "abstract_algebra"
+"task": "mmlu_pro_biology_generative"
+"task_alias": "biology"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_business.yaml

+"dataset_name": "business"
+"description": "The following are multiple choice questions (with answers) about business.\n\
+  \n"
+"group": "mmlu_pro_other_generative"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_business_generative"
+"task_alias": "business"

lm_eval/tasks/mmlu_pro/default/mmlu_anatomy.yaml → lm_eval/tasks/mmlu_pro/generative/mmlu_pro_chemistry.yaml

-"dataset_name": "anatomy"
-"description": "The following are multiple choice questions (with answers) about anatomy.\n\
+"dataset_name": "math"
+"description": "The following are multiple choice questions (with answers) about math.\n\
   \n"
-"group": "mmlu_stem"
+"group": "mmlu_pro_stem_generative"
 "group_alias": "stem"
 "include": "_default_template_yaml"
-"task": "mmlu_anatomy"
-"task_alias": "anatomy"
+"task": "mmlu_pro_math_generative"
+"task_alias": "math"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_computer_science.yaml

+"dataset_name": "computer_science"
+"description": "The following are multiple choice questions (with answers) about computer_science.\n\
+  \n"
+"group": "mmlu_pro_stem_generative"
+"group_alias": "stem"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_computer_science_generative"
+"task_alias": "computer_science"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_economics.yaml

+"dataset_name": "economics"
+"description": "The following are multiple choice questions (with answers) about economics.\n\
+  \n"
+"group": "mmlu_pro_social_sciences_generative"
+"group_alias": "social_sciences"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_economics_generative"
+"task_alias": "economics"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_engineering.yaml

+"dataset_name": "engineering"
+"description": "The following are multiple choice questions (with answers) about engineering.\n\
+  \n"
+"group": "mmlu_pro_stem_generative"
+"group_alias": "stem"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_engineering_generative"
+"task_alias": "engineering"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_health.yaml

+"dataset_name": "health"
+"description": "The following are multiple choice questions (with answers) about health.\n\
+  \n"
+"group": "mmlu_pro_other_generative"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_health_generative"
+"task_alias": "health"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_history.yaml

+"dataset_name": "history"
+"description": "The following are multiple choice questions (with answers) about history.\n\
+  \n"
+"group": "mmlu_pro_humanities_generative"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_history_generative"
+"task_alias": "history"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_law.yaml

+"dataset_name": "law"
+"description": "The following are multiple choice questions (with answers) about law.\n\
+  \n"
+"group": "mmlu_pro_humanities_generative"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_law_generative"
+"task_alias": "law"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_math.yaml

+"dataset_name": "chemistry"
+"description": "The following are multiple choice questions (with answers) about chemistry.\n\
+  \n"
+"group": "mmlu_pro_stem_generative"
+"group_alias": "stem"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_chemistry_generative"
+"task_alias": "chemistry"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_other.yaml

+"dataset_name": "other"
+"description": "The following are multiple choice questions (with answers) about other.\n\
+  \n"
+"group": "mmlu_pro_other_generative"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_other_generative"
+"task_alias": "other"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_philosophy.yaml

+"dataset_name": "philosophy"
+"description": "The following are multiple choice questions (with answers) about philosophy.\n\
+  \n"
+"group": "mmlu_pro_humanities_generative"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_philosophy_generative"
+"task_alias": "philosophy"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_physics.yaml

+"dataset_name": "physics"
+"description": "The following are multiple choice questions (with answers) about physics.\n\
+  \n"
+"group": "mmlu_pro_stem_generative"
+"group_alias": "stem"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_physics_generative"
+"task_alias": "physics"

lm_eval/tasks/mmlu_pro/generative/mmlu_pro_psychology.yaml

+"dataset_name": "psychology"
+"description": "The following are multiple choice questions (with answers) about psychology.\n\
+  \n"
+"group": "mmlu_pro_social_sciences_generative"
+"group_alias": "social_sciences"
+"include": "_default_template_yaml"
+"task": "mmlu_pro_psychology_generative"
+"task_alias": "psychology"