Merge branch 'main' into standardize_metrics

lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml

@@ -13,4 +13,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml

@@ -13,4 +13,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py

 import yaml
-import inspect
 import datasets
 
 from tqdm import tqdm
 
 
 def main() -> None:
-
     dataset_path = "EleutherAI/advanced_ai_risk"
     for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()):
         file_name = f"{task}.yaml"

lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml

@@ -11,4 +11,4 @@ doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}"
 metric_list:
   - metric: acc
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/model_written_evals/persona/_generate_configs.py

 import yaml
-import inspect
 import datasets
 
 from tqdm import tqdm
 
 
 def main() -> None:
-
     dataset_path = "EleutherAI/persona"
     for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()):
         file_name = f"{task}.yaml"

lm_eval/tasks/model_written_evals/persona/_template_yaml

@@ -9,4 +9,4 @@ doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml

@@ -11,4 +11,4 @@ doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml

@@ -11,4 +11,4 @@ doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

@@ -11,4 +11,4 @@ doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/model_written_evals/winogenerated/_template_yaml

@@ -9,4 +9,4 @@ doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/mutual/mutual.yaml

@@ -22,4 +22,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 2.0
+  version: 2.0

lm_eval/tasks/nq_open/nq_open.yaml

@@ -29,4 +29,4 @@ metric_list:
     regexes_to_ignore:
     - "\ban|a|the\b"
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/openbookqa/openbookqa.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/paws-x/_generate_config.py

 import argparse
-from typing import Dict, List
 
 import yaml
 

lm_eval/tasks/paws-x/pawsx_template_yaml

@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/pile/pile_arxiv.yaml

@@ -20,4 +20,4 @@ metric_list:
     aggregation: bits_per_byte
     higher_is_better: false
 metadata:
-  - version: 2.0
+  version: 2.0

lm_eval/tasks/piqa/piqa.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/polemo2/polemo2_in.yaml

@@ -42,4 +42,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/prost/corypaik_prost.yaml

@@ -16,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py

 def doc_to_text(doc) -> str:
     ctxs = "\n".join(doc["CONTEXTS"])
     return "Abstract: {}\nQuestion: {}\nAnswer:".format(
-        ctxs, doc["QUESTION"], doc["final_decision"]
+        ctxs,
+        doc["QUESTION"],
     )