Merge branch 'main' into standardize_metrics

lm_eval/tasks/coqa/default.yaml

@@ -19,4 +19,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 2.0
+  version: 2.0

lm_eval/tasks/coqa/utils.py

@@ -7,7 +7,7 @@ def doc_to_text(doc):
     # Given a passage p, the conversation history {q1, a1, . . . qi−1, ai−1}
     # and a question qi, the task is to predict the answer ai
     doc_text = doc["story"] + "\n\n"
-    for (q, a) in zip_longest(
+    for q, a in zip_longest(
         doc["questions"]["input_text"], doc["answers"]["input_text"][:-1]
     ):  # omit target answer ai
         question = f"Q: {q}\n\n"
@@ -17,7 +17,6 @@ def doc_to_text(doc):
 
 
 def doc_to_target(doc):
-
     turn_id = len(doc["questions"]["input_text"])
     # Returns unique answers and valid alternatives (Some questions in CoQA have multiple valid answers).
     answers = []
@@ -71,7 +70,6 @@ def compute_scores(gold_list, pred):
 
 
 def process_results(doc, results):
-
     gold_list = doc_to_target(doc)
     pred = results[0].strip().split("\n")[0]
 

lm_eval/tasks/crows_pairs/crows_pairs_english.yaml

@@ -20,4 +20,4 @@ metric_list:
     aggregation: mean
     higher_is_better: false
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/csatqa/_default_csatqa_yaml

@@ -14,4 +14,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/csatqa/_generate_configs.py

@@ -21,7 +21,6 @@ def parse_args():
 
 
 if __name__ == "__main__":
-
     args = parse_args()
 
     # get filename of base_yaml so we can `"include": ` it in our other YAMLs.
@@ -30,7 +29,6 @@ if __name__ == "__main__":
         base_yaml = yaml.full_load(f)
 
     for name in tqdm(SUBSETS):
-
         yaml_dict = {
             "include": base_yaml_name,
             "task": f"csatqa_{args.task_prefix}_{name}"

lm_eval/tasks/drop/default.yaml

@@ -21,4 +21,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 2.0
+  version: 2.0

lm_eval/tasks/drop/utils.py

@@ -62,7 +62,6 @@ def parse_answer(answer):
 
 
 def process_results(doc, results):
-
     preds, golds = results, doc["answers"]
     max_em = 0
     max_f1 = 0

lm_eval/tasks/fld/fld_default.yaml

@@ -12,3 +12,10 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+metadata:
+  version: 1.0

lm_eval/tasks/glue/cola/default.yaml

@@ -13,4 +13,4 @@ doc_to_decontamination_query: sentence
 metric_list:
   - metric: mcc
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/glue/mnli/default.yaml

@@ -11,4 +11,4 @@ doc_to_choice: ["True", "Neither", "False"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/glue/mrpc/default.yaml

@@ -12,4 +12,4 @@ metric_list:
   - metric: acc
   - metric: f1
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/glue/qnli/default.yaml

@@ -11,4 +11,4 @@ doc_to_choice: ["yes", "no"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/glue/qqp/default.yaml

@@ -12,4 +12,4 @@ metric_list:
   - metric: acc
   - metric: f1
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/glue/rte/default.yaml

@@ -11,4 +11,4 @@ doc_to_choice: ["True", "False"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/glue/sst2/default.yaml

@@ -11,4 +11,4 @@ doc_to_choice: ["negative", "positive"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/glue/wnli/default.yaml

@@ -11,4 +11,4 @@ doc_to_choice: ["False", "True"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 2.0
+  version: 2.0

lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml

@@ -31,4 +31,4 @@ filter_list:
       - function: "majority_vote"
       - function: "take_first"
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/gsm8k/gsm8k-cot.yaml

@@ -41,4 +41,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)."
       - function: "take_first"
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/gsm8k/gsm8k.yaml

@@ -34,4 +34,4 @@ filter_list:
         regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/headqa/headqa_en.yaml

@@ -20,4 +20,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0