add llama 3.1 mmlu

lm_eval/tasks/mmlu/generative/_default_template_yaml

-# dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
-# test_split: test
-# fewshot_split: dev
-# fewshot_config:
-#   sampler: first_n
-# output_type: generate_until
-# doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
-# doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}"
-# generation_kwargs:
-#   until:
-#     - "</s>"
-#     - "\n"
-# metric_list:
-#   - metric: exact_match
-#     aggregation: mean
-#     higher_is_better: true
-# metadata:
-#   version: 2.0
-# dataset_kwargs:
-#   trust_remote_code: true
 dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
 test_split: test
 fewshot_split: dev
 fewshot_config:
   sampler: first_n
 output_type: generate_until
-doc_to_text: 'Given the following question and four candidate answers (A, B, C and D), choose the best answer.\nQuestion: {{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{{{choices[3]}}}}\nYour response should end with \"The best answer is [the_answer_letter]\" where the [the_answer_letter] is one of A, B, C or D.'
+doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
 doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}"
-gen_prefix: "The best answer is"
 generation_kwargs:
   until:
     - "</s>"
     - "\n"
-filter:
-  - function: "regex"
-    group_select: -1
-    regex_pattern: 'best answer is ([A-Z])'
-  - function: "take_first"
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/mmlu/generative_llama_3_1_instruct/_default_template_yaml

+dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: generate_until
+doc_to_text: 'Given the following question and four candidate answers (A, B, C and D), choose the best answer.\nQuestion: {{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nYour response should end with \"The best answer is [the_answer_letter]\" where the [the_answer_letter] is one of A, B, C or D.'
+doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}"
+gen_prefix: "The best answer is"
+generation_kwargs:
+  max_gen_toks: 10
+filter_list:
+- filter:
+  - function: regex
+    group_select: -1
+    regex_pattern: (?:best answer is )?([A-Z])
+  - function: take_first
+  name: strict-match
+metric_list:
+  - metric: exact_match
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 2.0
+dataset_kwargs:
+  trust_remote_code: true