Merge branch 'main' into standardize_metrics

lm_eval/tasks/super_glue/copa/t5-prompt.yaml

@@ -19,4 +19,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/super_glue/multirc/default.yaml

@@ -12,4 +12,4 @@ doc_to_choice: "['''{{answer}}\\nIs the answer correct? yes''', '''{{answer}}\\n
 metric_list:
   - metric: acc
 metadata:
-  - version: 2.0
+  version: 2.0

lm_eval/tasks/super_glue/multirc/t5-prompt.yaml

@@ -20,4 +20,4 @@ metric_list:
     aggregation: !function t5_utils.agg_em
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/super_glue/multirc/t5_utils.py

@@ -5,7 +5,6 @@ import sklearn.metrics
 
 
 def f1(predictions, references):  # This is a passthrough function
-
     _prediction = predictions[0]
     _reference = references[0].split("_")[-1]
     string_label = ["False", "True"]
@@ -20,7 +19,6 @@ def f1(predictions, references):  # This is a passthrough function
 
 
 def agg_f1(items):
-
     predictions, references = zip(*items)
     references, predictions = np.asarray(references), np.asarray(predictions)
 
@@ -28,7 +26,6 @@ def agg_f1(items):
 
 
 def em(predictions, references):  # This is a passthrough function
-
     _prediction = predictions[0]
     _group, _reference = references[0].split("_")
     string_label = ["False", "True"]

lm_eval/tasks/super_glue/record/default.yaml

@@ -17,4 +17,4 @@ metric_list:
     higher_is_better: True
     aggregation: mean
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/super_glue/record/t5-prompt.yaml

@@ -19,4 +19,4 @@ metric_list:
     aggregation: !function t5_utils.squad_f1_agg
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/super_glue/record/t5_utils.py

@@ -3,14 +3,12 @@ import string
 import collections
 import numpy as np
 
-from tqdm import tqdm
-from datasets import Dataset, concatenate_datasets
+from datasets import Dataset
 
 from lm_eval.api.metrics import metric_max_over_ground_truths
 
 
 def doc_to_text(doc):
-
     passage = doc["passage"]
     passage = re.sub(r"(\.|\?|\!|\"|\')\n@highlight\n", r"\1 ", passage)
     passage = re.sub(r"\n@highlight\n", ". ", passage)
@@ -34,7 +32,6 @@ def process_docs(dataset):
         }
         answers = doc.pop("answers")
         for idx, answer in enumerate(answers):
-
             for key in split_doc.keys():
                 if key in doc:
                     split_doc[key].append(doc[key])

lm_eval/tasks/super_glue/rte/default.yaml

@@ -12,4 +12,4 @@ doc_to_choice: ['True', 'False']
 metric_list:
   - metric: acc
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/super_glue/rte/t5-prompt.yaml

@@ -19,4 +19,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/super_glue/wic/default.yaml

@@ -12,4 +12,4 @@ doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/super_glue/wic/t5-prompt.yaml

@@ -19,4 +19,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/super_glue/wsc/default.yaml

@@ -12,4 +12,4 @@ doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/super_glue/wsc/t5-prompt.yaml

@@ -20,4 +20,4 @@ filter_list:
     filter:
       - function: !function t5_utils.WSCPostprocess
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/super_glue/wsc/t5_utils.py

@@ -8,7 +8,6 @@ def doc_to_text(x):
 
 
 def _wsc_inputs(x):
-
     words = x["text"].split(" ")
 
     # We would need some special logic to handle the case where the pronoun is the
@@ -55,7 +54,6 @@ def _wsc_inputs(x):
 
 class WSCPostprocess(Filter):
     def __init__(self, **kwargs):
-
         self.determiners = {
             "a",
             "an",
@@ -86,10 +84,8 @@ class WSCPostprocess(Filter):
         return " ".join([w for w in s.split(" ") if w not in self.determiners])
 
     def apply(self, resps, docs):
-
         filtered_resps = []
         for prediction, reference in zip(*(resps, docs["span1_text"])):
-
             prediction = self.clean(prediction[0])
             reference = self.clean(reference)
 

lm_eval/tasks/swag/swag.yaml

@@ -16,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/toxigen/toxigen.yaml

@@ -15,4 +15,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/translation/utils.py

 import argparse
-from typing import Dict, List
 
 import yaml
 
-import sacrebleu
 
 try:
     import pycountry

lm_eval/tasks/translation/wmt_common_yaml

@@ -14,4 +14,4 @@ generation_kwargs:
   temperature: 0.0
 repeats: 1
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/triviaqa/default.yaml

@@ -28,4 +28,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  - version: 2.0
+  version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

@@ -76,4 +76,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 2.0
+  version: 2.0