fix bleu

lm_eval/tasks/super_glue/cb/t5-prompt.yaml

@@ -15,5 +15,6 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
-  - metric: f1
-    aggregation: !function "aggregate.cb_multi_fi"
+  - metric: !function "t5_utils.mean_3class_f1"
+    aggregation: !function "t5_utils.agg_mean_3class_f1"
+    higher_is_better: true

lm_eval/tasks/wmt2016/metrics.py

 import evaluate
 
-def bleu(references, predictions):
-    bleu_fn = evaluate.load('bleu')
-    return bleu_fn.compute(predictions=predictions, references=references)
+
+def bleu(predictions, references):
+    return (predictions[0], references[0])
+
+
+def agg_bleu(items):
+    bleu_fn = evaluate.load("bleu")
+    predictions, references = zip(*items)
+    return bleu_fn.compute(predictions=predictions, references=references)["bleu"]

lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml

@@ -13,5 +13,5 @@ metric_list:
     aggregation: mean
     higher_is_better: false
   - metric: !function metrics.bleu
-    aggregation: mean
+    aggregation: !function metrics.agg_bleu
     higher_is_better: true