revert parts back to main

lm_eval/api/metrics.py

@@ -256,7 +256,7 @@ def mcc_fn(items):  # This is a passthrough function
 @register_metric(
     metric="f1",
     higher_is_better=True,
-    output_type=["multiple_choice"],
+    output_type="multiple_choice",
     aggregation="f1",
 )
 def f1_fn(items):  # This is a passthrough function

lm_eval/api/task.py

@@ -1190,6 +1190,7 @@ class ConfigurableTask(Task):
                 eval_logger.warning("Applied prompt returns empty string")
                 return self.config.fewshot_delimiter
         else:
+            print(type(doc_to_text))
             raise TypeError
 
     def doc_to_target(self, doc: Mapping) -> Union[int, str, list]:
@@ -1279,6 +1280,7 @@ class ConfigurableTask(Task):
             else:
                 # Otherwise they are placed in the continuation
                 arguments = [(ctx, f"{target_delimiter}{cont}") for cont in choices]
+
             request_list = [
                 Instance(
                     request_type="loglikelihood",
@@ -1432,6 +1434,7 @@ class ConfigurableTask(Task):
                 ]
                 acc_mutual_info = 1.0 if np.argmax(lls_mutual_info) == gold else 0.0
                 result_dict["acc_mutual_info"] = acc_mutual_info
+
         elif self.OUTPUT_TYPE == "generate_until":
             gold = self.doc_to_target(doc)
             result = results[0]
@@ -1455,6 +1458,7 @@ class ConfigurableTask(Task):
                     scores = []
                     if not isinstance(gold, list):
                         # sometimes, a multiple_target dataset has exceptions where one doc has only one string answer
+                        # print(gold)
                         gold = [gold]
                     if metric == "exact_match":
                         result = [result for _ in range(len(gold))]
@@ -1489,10 +1493,10 @@ class ConfigurableTask(Task):
                 else:
                     try:
                         result_score = self._metric_fn_list[metric](
-                                references=[gold],
-                                predictions=[result],
-                                **self._metric_fn_kwargs[metric],
-                            )
+                            references=[gold],
+                            predictions=[result],
+                            **self._metric_fn_kwargs[metric],
+                        )
                     except TypeError:  # needed for now in order to use a different interface between our own metrics and HF Evaluate metrics
                         result_score = self._metric_fn_list[metric]([gold, result])
                     if isinstance(result_score, dict):
@@ -1646,4 +1650,4 @@ class PerplexityTask(Task):
     @classmethod
     def count_words(cls, doc) -> int:
         """Downstream tasks with custom word boundaries should override this!"""
-        return len(re.split(r"\s+", doc))
+        return len(re.split(r"\s+", doc))
\ No newline at end of file

lm_eval/filters/extraction.py

@@ -182,4 +182,4 @@ class MultiChoiceRegexFilter(RegexFilter):
                 filtered.append(match)
             filtered_resps.append(filtered)
 
-        return filtered_resps
+        return filtered_resps
\ No newline at end of file