changed default to 5-shot

lm_eval/tasks/benchmarks/flan/flan_held_out.yaml

@@ -3,10 +3,11 @@ task:
   # BBH
   - bbh_flan_zeroshot
   - bbh_flan_fewshot
-  # - bbh_flan_cot_fewshot
-  # - bbh_flan_cot_zeroshot
+  - bbh_flan_cot_fewshot
+  - bbh_flan_cot_zeroshot
   # MMLU
+  - mmlu
   - mmlu_flan_n_shot_generative
   - mmlu_flan_n_shot_loglikelihood
-  # - mmlu_flan_cot_zeroshot
-  # - mmlu_flan_cot_fewshot
+  - mmlu_flan_cot_zeroshot
+  - mmlu_flan_cot_fewshot

lm_eval/tasks/mmlu/default/_default_template_yaml

@@ -15,3 +15,4 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+num_fewshot: 5

lm_eval/tasks/mmlu/flan_n_shot/_mmlu_flan_generative_template_yaml

@@ -12,3 +12,4 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+num_fewshot: 5

lm_eval/tasks/mmlu/flan_n_shot/_mmlu_flan_loglikelihood_template_yaml

@@ -13,3 +13,4 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+num_fewshot: 5