New arabicmmlu (#2541)

* point to the original ArabicMMLU dataset

* create the new subtasks files

* fix bug when the context filed is empty

lm_eval/tasks/arabicmmlu/_arabicmmlu.yaml

@@ -9,4 +9,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_humanities.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_language.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_other.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_social_science.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_stem.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_default_arabicmmlu_template_yaml

-dataset_path: yazeed7/ArabicMMLU
+dataset_path: MBZUAI/ArabicMMLU
 test_split: test
 fewshot_split: dev
 fewshot_config:
@@ -12,4 +12,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/arabicmmlu/_generate_configs.py

@@ -13,48 +13,46 @@ from tqdm import tqdm
 eval_logger = logging.getLogger("lm-eval")
 
 
-SUBJECTS = {
-    "Driving Test": "other",
-    "High Geography": "social_science",
-    "High History": "humanities",
-    "Islamic Studies": "humanities",
-    "Univ Accounting": "social_science",
-    "Primary General Knowledge": "other",
-    "Univ Political Science": "social_science",
-    "Primary Math": "stem",
-    "Middle General Knowledge": "other",
-    "High Biology": "stem",
-    "Primary Natural Science": "stem",
-    "High Economics": "social_science",
-    "Middle Natural Science": "stem",
-    "Middle Geography": "social_science",
-    "Primary Social Science": "social_science",
-    "Middle Computer Science": "stem",
-    "Middle Islamic Studies": "humanities",
-    "Primary Computer Science": "stem",
-    "High Physics": "stem",
-    "Middle Social Science": "social_science",
-    "Middle Civics": "social_science",
-    "High Computer Science": "stem",
-    "General Knowledge": "other",
-    "High Civics": "social_science",
-    "Prof Law": "humanities",
-    "High Islamic Studies": "humanities",
-    "Primary Arabic Language": "language",
-    "High Arabic Language": "language",
-    "Arabic Language (Grammar)": "language",
-    "Primary History": "humanities",
-    "Middle History": "humanities",
-    "Univ Economics": "social_science",
-    "Arabic Language (General)": "language",
-    "Univ Computer Science": "stem",
-    "Primary Islamic Studies": "humanities",
-    "Primary Geography": "social_science",
-    "High Philosophy": "humanities",
-    "Middle Arabic Language": "language",
-    "Middle Economics": "social_science",
-    "Univ Management": "other",
-}
+SUBJECTS = {'Islamic Studies': 'humanities',
+ 'Driving Test': 'other',
+ 'Natural Science (Middle School)': 'stem',
+ 'Natural Science (Primary School)': 'stem',
+ 'History (Primary School)': 'humanities',
+ 'History (Middle School)': 'humanities',
+ 'History (High School)': 'humanities',
+ 'General Knowledge': 'other',
+ 'General Knowledge (Primary School)': 'other',
+ 'General Knowledge (Middle School)': 'other',
+ 'Law (Professional)': 'humanities',
+ 'Physics (High School)': 'stem',
+ 'Social Science (Middle School)': 'social_science',
+ 'Social Science (Primary School)': 'social_science',
+ 'Management (University)': 'other',
+ 'Arabic Language (Primary School)': 'language',
+ 'Arabic Language (Middle School)': 'language',
+ 'Arabic Language (High School)': 'language',
+ 'Political Science (University)': 'social_science',
+ 'Philosophy (High School)': 'humanities',
+ 'Accounting (University)': 'social_science',
+ 'Computer Science (University)': 'stem',
+ 'Computer Science (Middle School)': 'stem',
+ 'Computer Science (Primary School)': 'stem',
+ 'Computer Science (High School)': 'stem',
+ 'Geography (Primary School)': 'social_science',
+ 'Geography (Middle School)': 'social_science',
+ 'Geography (High School)': 'social_science',
+ 'Math (Primary School)': 'stem',
+ 'Biology (High School)': 'stem',
+ 'Economics (University)': 'social_science',
+ 'Economics (Middle School)': 'social_science',
+ 'Economics (High School)': 'social_science',
+ 'Arabic Language (General)': 'language',
+ 'Arabic Language (Grammar)': 'language',
+ 'Islamic Studies (High School)': 'humanities',
+ 'Islamic Studies (Middle School)': 'humanities',
+ 'Islamic Studies (Primary School)': 'humanities',
+ 'Civics (Middle School)': 'social_science',
+ 'Civics (High School)': 'social_science'}
 
 
 def parse_args():
@@ -69,8 +67,9 @@ if __name__ == "__main__":
 
     # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs.
     base_yaml_name = os.path.split(args.base_yaml_path)[-1]
-    with open(args.base_yaml_path, encoding="utf-8") as f:
-        base_yaml = yaml.full_load(f)
+
+    # with open(args.base_yaml_path, encoding="utf-8") as f:
+    #     base_yaml = yaml.full_load(f)
 
     ALL_CATEGORIES = []
     for subject, category in tqdm(SUBJECTS.items()):
@@ -81,8 +80,8 @@ if __name__ == "__main__":
 
         yaml_dict = {
             "include": base_yaml_name,
-            "tag": f"arabicmmlu_{category}",
-            "task": f"arabicmmlu_{subject.lower().replace(' ', '_')}",
+            "tag": f"arabicmmlu_{category}_tasks",
+            "task": f"arabicmmlu_{subject.lower().replace(' ', '_').replace('(', '').replace(')', '')}",
             "task_alias": subject,
             "dataset_name": subject,
             # "description": description,

lm_eval/tasks/arabicmmlu/arabicmmlu_middle_civics.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_accounting_university.yaml

-"dataset_name": "Middle Civics"
-"tag": "arabicmmlu_social_science_tasks"
+"dataset_name": "Accounting (University)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_middle_civics"
-"task_alias": "Middle Civics"
+"tag": "arabicmmlu_social_science_tasks"
+"task": "arabicmmlu_accounting_university"
+"task_alias": "Accounting (University)"

lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml

 "dataset_name": "Arabic Language (General)"
-"tag": "arabicmmlu_language_tasks"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_arabic_language_(general)"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_general"
 "task_alias": "Arabic Language (General)"

lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_grammar.yaml

 "dataset_name": "Arabic Language (Grammar)"
-"tag": "arabicmmlu_language_tasks"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_arabic_language_(grammar)"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_grammar"
 "task_alias": "Arabic Language (Grammar)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_arabic_language.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_high_school.yaml

-"dataset_name": "High Arabic Language"
-"tag": "arabicmmlu_language_tasks"
+"dataset_name": "Arabic Language (High School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_arabic_language"
-"task_alias": "High Arabic Language"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_high_school"
+"task_alias": "Arabic Language (High School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_middle_arabic_language.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_middle_school.yaml

-"dataset_name": "Middle Arabic Language"
-"tag": "arabicmmlu_language_tasks"
+"dataset_name": "Arabic Language (Middle School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_middle_arabic_language"
-"task_alias": "Middle Arabic Language"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_middle_school"
+"task_alias": "Arabic Language (Middle School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_primary_school.yaml

-"dataset_name": "Primary Arabic Language"
-"tag": "arabicmmlu_language_tasks"
+"dataset_name": "Arabic Language (Primary School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_primary_arabic_language"
-"task_alias": "Primary Arabic Language"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_primary_school"
+"task_alias": "Arabic Language (Primary School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_physics.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_biology_high_school.yaml

-"dataset_name": "High Physics"
-"tag": "arabicmmlu_stem_tasks"
+"dataset_name": "Biology (High School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_physics"
-"task_alias": "High Physics"
+"tag": "arabicmmlu_stem_tasks"
+"task": "arabicmmlu_biology_high_school"
+"task_alias": "Biology (High School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_economics.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_civics_high_school.yaml

-"dataset_name": "High Economics"
-"tag": "arabicmmlu_social_science_tasks"
+"dataset_name": "Civics (High School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_economics"
-"task_alias": "High Economics"
+"tag": "arabicmmlu_social_science_tasks"
+"task": "arabicmmlu_civics_high_school"
+"task_alias": "Civics (High School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_geography.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_civics_middle_school.yaml

-"dataset_name": "High Geography"
-"tag": "arabicmmlu_social_science_tasks"
+"dataset_name": "Civics (Middle School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_geography"
-"task_alias": "High Geography"
+"tag": "arabicmmlu_social_science_tasks"
+"task": "arabicmmlu_civics_middle_school"
+"task_alias": "Civics (Middle School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_computer_science.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_computer_science_high_school.yaml

-"dataset_name": "High Computer Science"
-"tag": "arabicmmlu_stem_tasks"
+"dataset_name": "Computer Science (High School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_computer_science"
-"task_alias": "High Computer Science"
+"tag": "arabicmmlu_stem_tasks"
+"task": "arabicmmlu_computer_science_high_school"
+"task_alias": "Computer Science (High School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_computer_science_middle_school.yaml

+"dataset_name": "Computer Science (Middle School)"
+"include": "_default_arabicmmlu_template_yaml"
+"tag": "arabicmmlu_stem_tasks"
+"task": "arabicmmlu_computer_science_middle_school"
+"task_alias": "Computer Science (Middle School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_computer_science_primary_school.yaml

+"dataset_name": "Computer Science (Primary School)"
+"include": "_default_arabicmmlu_template_yaml"
+"tag": "arabicmmlu_stem_tasks"
+"task": "arabicmmlu_computer_science_primary_school"
+"task_alias": "Computer Science (Primary School)"