Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into group-agg-rework

lm_eval/tasks/arabicmmlu/arabicmmlu.yaml → lm_eval/tasks/arabicmmlu/_arabicmmlu.yaml

@@ -5,3 +5,8 @@ task:
 - arabicmmlu_humanities
 - arabicmmlu_stem
 - arabicmmlu_language
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_humanities.yaml

+group: arabicmmlu_humanities
+group_alias: humanities
+task:
+  - arabicmmlu_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_language.yaml

+group: arabicmmlu_language
+group_alias: language
+task:
+  - arabicmmlu_language_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_other.yaml

+group: arabicmmlu_other
+group_alias: other
+task:
+  - arabicmmlu_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_social_science.yaml

+group: arabicmmlu_social_science
+group_alias: social_science
+task:
+  - arabicmmlu_social_science_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_arabicmmlu_stem.yaml

+group: arabicmmlu_stem
+group_alias: stem
+task:
+  - arabicmmlu_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/arabicmmlu/_default_template_yaml

@@ -11,3 +11,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/arabicmmlu/_generate_configs.py

@@ -81,8 +81,7 @@ if __name__ == "__main__":
 
         yaml_dict = {
             "include": base_yaml_name,
-            "group": f"arabicmmlu_{category}",
-            "group_alias": category.replace("_", " "),
+            "tag": f"arabicmmlu_{category}",
             "task": f"arabicmmlu_{subject.lower().replace(' ', '_')}",
             "task_alias": subject,
             "dataset_name": subject,

lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml

 "dataset_name": "Arabic Language (General)"
-"group": "arabicmmlu_language"
-"group_alias": "language"
+"tag": "arabicmmlu_language_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_arabic_language_(general)"
 "task_alias": "Arabic Language (General)"

lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_grammar.yaml

 "dataset_name": "Arabic Language (Grammar)"
-"group": "arabicmmlu_language"
-"group_alias": "language"
+"tag": "arabicmmlu_language_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_arabic_language_(grammar)"
 "task_alias": "Arabic Language (Grammar)"

lm_eval/tasks/arabicmmlu/arabicmmlu_driving_test.yaml

 "dataset_name": "Driving Test"
-"group": "arabicmmlu_other"
-"group_alias": "other"
+"tag": "arabicmmlu_other_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_driving_test"
 "task_alias": "Driving Test"

lm_eval/tasks/arabicmmlu/arabicmmlu_general_knowledge.yaml

 "dataset_name": "General Knowledge"
-"group": "arabicmmlu_other"
-"group_alias": "other"
+"tag": "arabicmmlu_other_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_general_knowledge"
 "task_alias": "General Knowledge"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_arabic_language.yaml

 "dataset_name": "High Arabic Language"
-"group": "arabicmmlu_language"
-"group_alias": "language"
+"tag": "arabicmmlu_language_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_high_arabic_language"
 "task_alias": "High Arabic Language"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_biology.yaml

 "dataset_name": "High Biology"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
+"tag": "arabicmmlu_stem_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_high_biology"
 "task_alias": "High Biology"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_civics.yaml

 "dataset_name": "High Civics"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
+"tag": "arabicmmlu_social_science_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_high_civics"
 "task_alias": "High Civics"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_computer_science.yaml

 "dataset_name": "High Computer Science"
-"group": "arabicmmlu_stem"
-"group_alias": "stem"
+"tag": "arabicmmlu_stem_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_high_computer_science"
 "task_alias": "High Computer Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_economics.yaml

 "dataset_name": "High Economics"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
+"tag": "arabicmmlu_social_science_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_high_economics"
 "task_alias": "High Economics"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_geography.yaml

 "dataset_name": "High Geography"
-"group": "arabicmmlu_social_science"
-"group_alias": "social science"
+"tag": "arabicmmlu_social_science_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_high_geography"
 "task_alias": "High Geography"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_history.yaml

 "dataset_name": "High History"
-"group": "arabicmmlu_humanities"
-"group_alias": "humanities"
+"tag": "arabicmmlu_humanities_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_high_history"
 "task_alias": "High History"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_islamic_studies.yaml

 "dataset_name": "High Islamic Studies"
-"group": "arabicmmlu_humanities"
-"group_alias": "humanities"
+"tag": "arabicmmlu_humanities_tasks"
 "include": "_default_template_yaml"
 "task": "arabicmmlu_high_islamic_studies"
 "task_alias": "High Islamic Studies"