mmlu-pro: fixed doc_to_text/choice/target configs for all variants

lm_eval/tasks/mmlu_pro/default/_default_template_yaml

@@ -4,8 +4,8 @@ fewshot_split: dev
 fewshot_config:
   sampler: first_n
 output_type: multiple_choice
-doc_to_text: "{% set alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' %}{{ question.strip() }}{% for index in range(options|length) %}\n{{ alphabet[index] }}. {{ options[index] }}{% endfor %}\nAnswer:"
-doc_to_choice: "{% set alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' %}{% set choices = [] %}{% for index in range(options|length) %}{% set _ = choices.append(alphabet[index]) %}{% endfor %}{{ choices }}"
+doc_to_text: "{% set alphabet = 'ABCDEFGHIJ' %}{{ question.strip() }}{% for index in range(options|length) %}\n{{ alphabet[index] }}. {{ options[index] }}{% endfor %}\nAnswer:"
+doc_to_choice: "{% set alphabet = 'ABCDEFGHIJ' %}{% set choices = [] %}{% for index in range(options|length) %}{% set _ = choices.append(alphabet[index]) %}{% endfor %}{{ choices }}"
 doc_to_target: answer_index
 metric_list:
   - metric: acc

lm_eval/tasks/mmlu_pro/flan_cot_fewshot/_mmlu_pro_flan_cot_fewshot_template_yaml

@@ -5,7 +5,7 @@ fewshot_split: dev
 fewshot_config:
   sampler: first_n
 output_type: generate_until
-doc_to_text: "Q: {{question.strip()}}\n(A) {{options[0]}} (B) {{options[1]}} (C) {{options[2]}} (D) {{options[3]}} (E) {{options[4]}} (F) {{options[5]}} (G) {{options[6]}} (H) {{options[7]}} (I) {{options[8]}} (J) {{options[9]}}\nA: Let's think step by step."
+doc_to_text: "{% set alphabet = 'ABCDEFGHIJ' %}{{ question.strip() }}\n{% for index in range(options|length) %}({{ alphabet[index] }}) {{ options[index] }} {% endfor %}\nA: Let's think step by step."
 doc_to_target: "{{['(A)', '(B)', '(C)', '(D)', '(E)', '(F)', '(G)', '(H)', '(I)', '(J)'][answer_index]}}"
 filter_list:
   - name: "get-answer"

lm_eval/tasks/mmlu_pro/flan_cot_zeroshot/_mmlu_pro_flan_cot_zeroshot_template_yaml

@@ -2,7 +2,7 @@ dataset_path: sjyuxyz/MMLU-Pro-with-subset
 validation_split: validation
 fewshot_split: dev
 output_type: generate_until
-doc_to_text: "Q: {{question.strip()}}\n(A) {{options[0]}} (B) {{options[1]}} (C) {{options[2]}} (D) {{options[3]}} (E) {{options[4]}} (F) {{options[5]}} (G) {{options[6]}} (H) {{options[7]}} (I) {{options[8]}} (J) {{options[9]}}\nA: Let's think step by step."
+doc_to_text: "{% set alphabet = 'ABCDEFGHIJ' %}{{ question.strip() }}\n{% for index in range(options|length) %}({{ alphabet[index] }}) {{ options[index] }} {% endfor %}\nA: Let's think step by step."
 doc_to_target: "{{['(A)', '(B)', '(C)', '(D)', '(E)', '(F)', '(G)', '(H)', '(I)', '(J)'][answer]}}"
 filter_list:
   - name: "strict-match"

lm_eval/tasks/mmlu_pro/generative/_default_template_yaml

@@ -4,8 +4,8 @@ fewshot_split: dev
 fewshot_config:
   sampler: first_n
 output_type: generate_until
-doc_to_text: "{{question.strip()}}\nA. {{options[0]}}\nB. {{options[1]}}\nC. {{options[2]}}\nD. {{options[3]}}\nE. {{options[4]}}\nF. {{options[5]}}\nG. {{options[6]}}\nH. {{options[7]}}\nI. {{options[8]}}\nJ. {{options[9]}}\nAnswer:"
-doc_to_target: "{{['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'][answer]}}"
+doc_to_text: "{% set alphabet = 'ABCDEFGHIJ' %}{{ question.strip() }}{% for index in range(options|length) %}\n{{ alphabet[index] }}. {{ options[index] }}{% endfor %}\nAnswer:"
+doc_to_target: "{{['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'][answer_index]}}"
 generation_kwargs:
   until:
     - "</s>"