mmlu-pro: added continuation and flan_cot_zeroshot

lm_eval/tasks/mmlu_pro/continuation/_continuation_template_yaml

+dataset_path: sjyuxyz/MMLU-Pro-with-subset
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+doc_to_text: "Question: {{question.strip()}}\nAnswer:"
+doc_to_choice: "{{choices}}"
+doc_to_target: "{{answer}}"
+metadata:
+  version: 0.0

lm_eval/tasks/mmlu_pro/continuation/_mmlu_pro.yaml

+group: mmlu_pro_continuation
+task:
+  - mmlu_pro_continuation_stem
+  - mmlu_pro_continuation_other
+  - mmlu_pro_continuation_social_sciences
+  - mmlu_pro_continuation_humanities

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_biology.yaml

+"dataset_name": "biology"
+"description": "The following are questions (with answers) about biology.\n\
+  \n"
+"group": "mmlu_continuation_stem"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_biology"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_business.yaml

+"dataset_name": "business"
+"description": "The following are multiple choice questions (with answers) about business.\n\
+  \n"
+"group": "mmlu_continuation_other"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_business"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_chemistry.yaml

+"dataset_name": "math"
+"description": "The following are multiple choice questions (with answers) about math.\n\
+  \n"
+"group": "mmlu_continuation_stem"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_chemistry"
\ No newline at end of file

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_computer_science.yaml

+"dataset_name": "computer_science"
+"description": "The following are multiple choice questions (with answers) about computer_science.\n\
+  \n"
+"group": "mmlu_continuation_stem"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_computer_science"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_economics.yaml

+"dataset_name": "economics"
+"description": "The following are multiple choice questions (with answers) about economics.\n\
+  \n"
+"group": "mmlu_continuation_social_sciences"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_economics"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_engineering.yaml

+"dataset_name": "engineering"
+"description": "The following are multiple choice questions (with answers) about engineering.\n\
+  \n"
+"group": "mmlu_continuation_stem"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_engineering"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_health.yaml

+"dataset_name": "health"
+"description": "The following are multiple choice questions (with answers) about health.\n\
+  \n"
+"group": "mmlu_continuation_other"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_health"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_history.yaml

+"dataset_name": "history"
+"description": "The following are multiple choice questions (with answers) about history.\n\
+  \n"
+"group": "mmlu_continuation_humanities"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_history"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_law.yaml

+"dataset_name": "law"
+"description": "The following are multiple choice questions (with answers) about law.\n\
+  \n"
+"group": "mmlu_continuation_humanities"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_law"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_math.yaml

+"dataset_name": "math"
+"description": "The following are multiple choice questions (with answers) about math.\n\
+  \n"
+"group": "mmlu_continuation_stem"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_math"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_other.yaml

+"dataset_name": "other"
+"description": "The following are multiple choice questions (with answers) about other.\n\
+  \n"
+"group": "mmlu_continuation_other"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_other"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_philosophy.yaml

+"dataset_name": "philosophy"
+"description": "The following are multiple choice questions (with answers) about philosophy.\n\
+  \n"
+"group": "mmlu_continuation_humanities"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_philosophy"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_physics.yaml

+"dataset_name": "physics"
+"description": "The following are multiple choice questions (with answers) about physics.\n\
+  \n"
+"group": "mmlu_continuation_stem"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_physics"

lm_eval/tasks/mmlu_pro/continuation/mmlu_pro_psychology.yaml

+"dataset_name": "psychology"
+"description": "The following are multiple choice questions (with answers) about psychology.\n\
+  \n"
+"group": "mmlu_continuation_social_sciences"
+"include": "_continuation_template_yaml"
+"task": "mmlu_continuation_psychology"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/_mmlu_pro.yaml

+group: mmlu_pro_flan_cot_zeroshot
+task:
+  - mmlu_pro_flan_cot_zeroshot_stem
+  - mmlu_pro_flan_cot_zeroshot_other
+  - mmlu_pro_flan_cot_zeroshot_social_sciences
+  - mmlu_pro_flan_cot_zeroshot_humanities

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/_mmlu_pro_flan_cot_zeroshot_template_yaml

+dataset_path: sjyuxyz/MMLU-Pro-with-subset
+validation_split: validation
+fewshot_split: dev
+output_type: generate_until
+doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}} (E) {{choices[4]}} (F) {{choices[5]}} (G) {{choices[6]}} (H) {{choices[7]}} (I) {{choices[8]}} (J) {{choices[9]}}\nA: Let's think step by step."
+doc_to_target: "{{['(A)', '(B)', '(C)', '(D)', '(E)', '(F)', '(G)', '(H)', '(I)', '(J)'][answer]}}"
+filter_list:
+  - name: "strict-match"
+    filter:
+      - function: "regex"
+        regex_pattern: "((?<=The answer is )(.*)(?=.)|(?<=answer is )(.*)(?=.)|(?<=The answer: )(.*)(?=.)|(?<=The final answer: )(.*)(?=.))"
+      - function: "take_first"
+  - name: "flexible-extract"
+    filter:
+      - function: !function utils.MultiChoiceRegexFilter
+        group_select: -1
+        ignore_case: true
+        ignore_punctuation: true
+        regex_pattern: "(\\([A-Z]\\))"
+      - function: "take_first"
+generation_kwargs:
+  until:
+    - "</s>"
+    - "Q:"
+    - "<|im_end|>"
+  do_sample: false
+  temperature: 0.0
+num_fewshot: 0
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 2.0

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_biology.yaml

+"dataset_name": "biology"
+"description": "The following are multiple choice questions (with answers) about biology.\n\
+  \n"
+"group": "mmlu_pro_flan_cot_zeroshot_stem"
+"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
+"task": "mmlu_pro_flan_cot_zeroshot_biology"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/mmlu_pro_business.yaml

+"dataset_name": "business"
+"description": "The following are multiple choice questions (with answers) about business.\n\
+  \n"
+"group": "mmlu_pro_flan_cot_zeroshot_other"
+"include": "_mmlu_pro_flan_cot_zeroshot_template_yaml"
+"task": "mmlu_pro_flan_cot_zeroshot_business"