Merge branch 'main' into standardize_metrics

lm_eval/tasks/kmmlu/utils.py

-import datasets
-
-
-def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
-    def _process_doc(doc):
-        instruction = (
-            f"다음을 읽고 정답으로 알맞은 것을 고르시요.\n"
-            f"### Question: {doc['question']}\n"
-            f"### Options:\n"
-            f"(1) {doc['option#1']}\n(2) {doc['option#2']}\n(3) {doc['option#3']}\n(4) {doc['option#4']}\n"
-            f"### Answer: 주어진 문제의 정답은"
-        )
-        out_doc = {
-            "question": instruction,
-            "choices": ["(1)", "(2)", "(3)", "(4)"],
-            "gold": int(doc["answer"]) - 1,
-        }
-        return out_doc
-
-    return dataset.map(_process_doc)

lm_eval/tasks/lambada/lambada_openai.yaml

@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/lambada/lambada_standard.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml

@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/lambada_cloze/lambada_standard_cloze.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/lambada_multilingual/lambada_mt_en.yaml

@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/logiqa/logiqa.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/logiqa2/logieval.yaml

@@ -24,4 +24,4 @@ filter_list:
         regex_pattern: "^\\s*([A-D])"
       - function: "take_first"
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/logiqa2/logiqa2.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/mathqa/mathqa.yaml

@@ -19,4 +19,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/mc_taco/default.yaml

@@ -12,4 +12,4 @@ metric_list:
   - metric: acc
   - metric: f1
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/mgsm/direct/direct_yaml

@@ -26,4 +26,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/mgsm/en_cot/cot_yaml

@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/mgsm/native_cot/cot_yaml

@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  - version: 1.0
+  version: 1.0

lm_eval/tasks/mgsm/utils.py

@@ -94,7 +94,6 @@ LANGUAGES = {
 
 
 def add_regex_pattern(regex_pattern):
-
     if regex_pattern is None:
         return {}
     return {

lm_eval/tasks/minerva_math/minerva_math_algebra.yaml

@@ -21,4 +21,4 @@ metric_list:
     higher_is_better: true
 num_fewshot: 0
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/mmlu/_generate_configs.py

@@ -7,7 +7,6 @@ import argparse
 
 from tqdm import tqdm
 
-from lm_eval import utils
 from lm_eval.logger import eval_logger
 
 SUBJECTS = {
@@ -82,7 +81,6 @@ def parse_args():
 
 
 if __name__ == "__main__":
-
     args = parse_args()
 
     # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs.
@@ -98,7 +96,6 @@ if __name__ == "__main__":
 
     ALL_CATEGORIES = []
     for subject, category in tqdm(SUBJECTS.items()):
-
         if category not in ALL_CATEGORIES:
             ALL_CATEGORIES.append(category)
 

lm_eval/tasks/mmlu/default/_default_template_yaml

@@ -12,4 +12,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml

@@ -23,4 +23,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
+  version: 0.0

lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml

@@ -23,4 +23,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
+  version: 0.0