process multiple_output

lm_eval/api/task.py

@@ -919,9 +919,6 @@ class ConfigurableTask(Task):
                 choices = self.doc_to_choice(doc)
             completion_len = np.array([float(len(i)) for i in choices])
 
-            if self.multiple_output:
-                pass
-
             if (
                 2 * len(choices) == len(lls)
                 and "acc_mutual_info" in self._metric_fn_list.keys()
@@ -946,12 +943,23 @@ class ConfigurableTask(Task):
                 pred = choices[pred_idx]
                 pred_norm = choices[pred_idx_norm]
                 gold = self.doc_to_target(doc)
+                if self.multiple_output:
+                    if type(gold[0]) == int:
+                        gold_idx = gold
+                        gold = [choices[idx] for idx in gold_idx]
+                    elif type(gold[0]) == str:
+                        gold_idx = [choices.index(g) for g in gold]
+                else:
                     if type(gold) == int:
                         gold_idx = gold
                         gold = choices[gold_idx]
                     elif type(gold) == str:
                         gold_idx = choices.index(gold)
 
+            if self.multiple_output:
+                acc = 1.0 if pred in gold else 0.0
+                acc_norm = 1.0 if pred_norm in gold else 0.0
+            else:
                 acc = 1.0 if pred == gold else 0.0
                 acc_norm = 1.0 if pred_norm == gold else 0.0