ONLY FOR DEFAULT: fixed yaml file to use variable number of choices

lm_eval/tasks/mmlu_pro/continuation/_continuation_template_yaml

@@ -5,6 +5,6 @@ fewshot_config:
   sampler: first_n
 doc_to_text: "Question: {{question.strip()}}\nAnswer:"
 doc_to_choice: "{{options}}"
-doc_to_target: "{{answer}}"
+doc_to_target: "{{answer_index}}"
 metadata:
   version: 0.0

lm_eval/tasks/mmlu_pro/default/_default_template_yaml

@@ -4,9 +4,9 @@ fewshot_split: dev
 fewshot_config:
   sampler: first_n
 output_type: multiple_choice
-doc_to_text: "{{question.strip()}}\nA. {{options[0]}}\nB. {{options[1]}}\nC. {{options[2]}}\nD. {{options[3]}}\nE. {{options[4]}}\nF. {{options[5]}}\nG. {{options[6]}}\nH. {{options[7]}}\nI. {{options[8]}}\nJ. {{options[9]}}\nAnswer:"
-doc_to_choice: ["A", "B", "C", "D", "E","F","G","H","I","J"]
-doc_to_target: answer
+doc_to_text: "{% set alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' %}{{ question.strip() }}{% for index in range(options|length) %}\n{{ alphabet[index] }}. {{ options[index] }}{% endfor %}\nAnswer:"
+doc_to_choice: "{% set alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' %}{% set choices = [] %}{% for index in range(options|length) %}{% set _ = choices.append(alphabet[index]) %}{% endfor %}{{ choices }}"
+doc_to_target: answer_index
 metric_list:
   - metric: acc
     aggregation: mean

lm_eval/tasks/mmlu_pro/flan_cot_fewshot/_mmlu_pro_flan_cot_fewshot_template_yaml

@@ -6,7 +6,7 @@ fewshot_config:
   sampler: first_n
 output_type: generate_until
 doc_to_text: "Q: {{question.strip()}}\n(A) {{options[0]}} (B) {{options[1]}} (C) {{options[2]}} (D) {{options[3]}} (E) {{options[4]}} (F) {{options[5]}} (G) {{options[6]}} (H) {{options[7]}} (I) {{options[8]}} (J) {{options[9]}}\nA: Let's think step by step."
-doc_to_target: "{{['(A)', '(B)', '(C)', '(D)', '(E)', '(F)', '(G)', '(H)', '(I)', '(J)'][answer]}}"
+doc_to_target: "{{['(A)', '(B)', '(C)', '(D)', '(E)', '(F)', '(G)', '(H)', '(I)', '(J)'][answer_index]}}"
 filter_list:
   - name: "get-answer"
     filter: