Merge branch 'upstream' into 'mmlu-pro'

add tokenizer logs info (#1731)

See merge request shijie.yu/lm-evaluation-harness!4

lm_eval/tasks/agieval/lsat-ar.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_lsat_ar
 dataset_path: hails/agieval-lsat-ar

lm_eval/tasks/agieval/lsat-lr.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_lsat_lr
 dataset_path: hails/agieval-lsat-lr

lm_eval/tasks/agieval/lsat-rc.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_lsat_rc
 dataset_path: hails/agieval-lsat-rc

lm_eval/tasks/agieval/math.yaml

-group:
-  - agieval
-  - agieval_en
 task: agieval_math
 dataset_path: hails/agieval-math
 dataset_name: null

lm_eval/tasks/agieval/sat-en-without-passage.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_sat_en_without_passage
 dataset_path: hails/agieval-sat-en-without-passage

lm_eval/tasks/agieval/sat-en.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_sat_en
 dataset_path: hails/agieval-sat-en

lm_eval/tasks/agieval/sat-math.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_sat_math
 dataset_path: hails/agieval-sat-math

lm_eval/tasks/anli/anli_r1.yaml

-group:
+tag:
   - anli
 task: anli_r1
 dataset_path: anli

lm_eval/tasks/arabicmmlu/arabicmmlu.yaml → lm_eval/tasks/arabicmmlu/_arabicmmlu.yaml

@@ -5,3 +5,8 @@ task:
 - arabicmmlu_humanities
 - arabicmmlu_stem
 - arabicmmlu_language
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_humanities.yaml

+group: arabicmmlu_humanities
+group_alias: Humanities
+task:
+  - arabicmmlu_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_language.yaml

+group: arabicmmlu_language
+group_alias: Language
+task:
+  - arabicmmlu_language_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_other.yaml

+group: arabicmmlu_other
+group_alias: Other
+task:
+  - arabicmmlu_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_social_science.yaml

+group: arabicmmlu_social_science
+group_alias: Social Science
+task:
+  - arabicmmlu_social_science_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_stem.yaml

+group: arabicmmlu_stem
+group_alias: STEM
+task:
+  - arabicmmlu_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_default_template_yaml → lm_eval/tasks/arabicmmlu/_default_arabicmmlu_template_yaml

@@ -11,3 +11,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/arabicmmlu/_generate_configs.py

 """
 Take in a YAML, and output all "other" splits with this YAML
 """
+
 import argparse
 import logging
 import os
@@ -58,7 +59,7 @@ SUBJECTS = {
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--base_yaml_path", default="_default_template_yaml")
+    parser.add_argument("--base_yaml_path", default="_default_arabicmmlu_template_yaml")
     parser.add_argument("--save_prefix_path", default="arabicmmlu")
     return parser.parse_args()
 
@@ -76,20 +77,21 @@ if __name__ == "__main__":
         if category not in ALL_CATEGORIES:
             ALL_CATEGORIES.append(category)
 
-        
         # description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n"
 
         yaml_dict = {
             "include": base_yaml_name,
-            "group": f"arabicmmlu_{category}",
-            "group_alias": category.replace("_", " "),
+            "tag": f"arabicmmlu_{category}",
             "task": f"arabicmmlu_{subject.lower().replace(' ', '_')}",
             "task_alias": subject,
             "dataset_name": subject,
             # "description": description,
         }
 
-        file_save_path = args.save_prefix_path + f"_{subject.lower().replace(' ', '_').replace('(', '').replace(')', '')}.yaml"
+        file_save_path = (
+            args.save_prefix_path
+            + f"_{subject.lower().replace(' ', '_').replace('(', '').replace(')', '')}.yaml"
+        )
         eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}")
         with open(file_save_path, "w", encoding="utf-8") as yaml_file:
             yaml.dump(

lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml

 "dataset_name": "Arabic Language (General)"
-"group": "arabicmmlu_language"
-"group_alias": "language"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_language_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_arabic_language_(general)"
 "task_alias": "Arabic Language (General)"

lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_grammar.yaml

 "dataset_name": "Arabic Language (Grammar)"
-"group": "arabicmmlu_language"
-"group_alias": "language"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_language_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_arabic_language_(grammar)"
 "task_alias": "Arabic Language (Grammar)"

lm_eval/tasks/arabicmmlu/arabicmmlu_driving_test.yaml

 "dataset_name": "Driving Test"
-"group": "arabicmmlu_other"
-"group_alias": "other"
-"include": "_default_template_yaml"
+"tag": "arabicmmlu_other_tasks"
+"include": "_default_arabicmmlu_template_yaml"
 "task": "arabicmmlu_driving_test"
 "task_alias": "Driving Test"