adjustments

lm_eval/prompts/__init__.py

@@ -116,6 +116,11 @@ class PromptString:
 
         doc_to_text = self.prompt_string["doc_to_text"]
         doc_to_target = self.prompt_string["doc_to_target"]
+
+        # TODO need a way to process doc_to_choice
+        if "doc_to_choice" in self.prompt_string:
+            raise "Not yet implemented to accept doc_to_choice"
+
         text_string = utils.apply_template(doc_to_text, doc)
         target_string = utils.apply_template(doc_to_target, doc)
 

lm_eval/tasks/mmlu/default/_default_template_yaml

@@ -15,4 +15,3 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
-num_fewshot: 5

lm_eval/tasks/mmlu/flan_n_shot/_mmlu_flan_generative_template_yaml

@@ -12,4 +12,3 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
-num_fewshot: 5

lm_eval/tasks/mmlu/flan_n_shot/_mmlu_flan_loglikelihood_template_yaml

@@ -13,4 +13,3 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
-num_fewshot: 5