Merge branch 'main' into humaneval

# Conflicts:
#	lm_eval/api/task.py

lm_eval/tasks/arabicmmlu/arabicmmlu_middle_social_science.yaml

+"dataset_name": "Middle Social Science"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_middle_social_science"
+"task_alias": "Middle Social Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml

+"dataset_name": "Primary Arabic Language"
+"tag": "arabicmmlu_language_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_arabic_language"
+"task_alias": "Primary Arabic Language"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_computer_science.yaml

+"dataset_name": "Primary Computer Science"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_computer_science"
+"task_alias": "Primary Computer Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_general_knowledge.yaml

+"dataset_name": "Primary General Knowledge"
+"tag": "arabicmmlu_other_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_general_knowledge"
+"task_alias": "Primary General Knowledge"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_geography.yaml

+"dataset_name": "Primary Geography"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_geography"
+"task_alias": "Primary Geography"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_history.yaml

+"dataset_name": "Primary History"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_history"
+"task_alias": "Primary History"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_islamic_studies.yaml

+"dataset_name": "Primary Islamic Studies"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_islamic_studies"
+"task_alias": "Primary Islamic Studies"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_math.yaml

+"dataset_name": "Primary Math"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_math"
+"task_alias": "Primary Math"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_natural_science.yaml

+"dataset_name": "Primary Natural Science"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_natural_science"
+"task_alias": "Primary Natural Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_social_science.yaml

+"dataset_name": "Primary Social Science"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_social_science"
+"task_alias": "Primary Social Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_prof_law.yaml

+"dataset_name": "Prof Law"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_prof_law"
+"task_alias": "Prof Law"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_accounting.yaml

+"dataset_name": "Univ Accounting"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_univ_accounting"
+"task_alias": "Univ Accounting"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_computer_science.yaml

+"dataset_name": "Univ Computer Science"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_univ_computer_science"
+"task_alias": "Univ Computer Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_economics.yaml

+"dataset_name": "Univ Economics"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_univ_economics"
+"task_alias": "Univ Economics"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml

+"dataset_name": "Univ Management"
+"tag": "arabicmmlu_other_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_univ_management"
+"task_alias": "Univ Management"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml

+"dataset_name": "Univ Political Science"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_univ_political_science"
+"task_alias": "Univ Political Science"

lm_eval/tasks/arabicmmlu/utils.py

+PROMPT = "This is a {}. Select the correct answer!\n\nQuestion: {}\n{}\n\nAnswer:"
+
+level_en = {
+    "Primary": "primary school",
+    "Middle": "middle school",
+    "High": "high school",
+    "Univ": "university",
+    "Prof": "professional",
+}
+
+alpa = ["A.", "B.", "C.", "D.", "E."]
+
+
+def doc_to_text(doc):
+    """
+    Refactoring `prepare_data_en` to fit with the lm harness framework.
+    https://github.com/mbzuai-nlp/ArabicMMLU/blob/main/util_prompt.py
+    """
+
+    level = "" if not doc["Level"] else " for " + level_en[doc["Level"]]
+    country = "" if not doc["Country"] else " in " + doc["Country"]
+    main_meta_data = f"{doc['Subject']} question{level}{country}"
+
+    question = (
+        doc["Question"]
+        if doc["Context"] == ""
+        else f"{doc['Context']}\n\n{doc['Question']}"
+    )
+
+    options = []
+    for i, opt in enumerate(
+        ["Option 1", "Option 2", "Option 3", "Option 4", "Option 5"]
+    ):
+        if not doc[opt]:
+            break
+        options.append(f"{alpa[i]} {doc[opt]}")
+
+    doc_text = PROMPT.format(main_meta_data, question, "\n".join(options))
+
+    return doc_text
+
+
+def doc_to_choice(doc):
+    return [alpa[i][0] for i in range(5) if doc[f"Option {i+1}"]]

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU.yaml

+group: AraDiCE_ArabicMMLU_egy
+task:
+- AraDiCE_ArabicMMLU_humanities_egy
+- AraDiCE_ArabicMMLU_language_egy
+- AraDiCE_ArabicMMLU_social-science_egy
+- AraDiCE_ArabicMMLU_stem_egy
+- AraDiCE_ArabicMMLU_other_egy
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+  - metric: acc_norm
+    weight_by_size: True

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_humanities_history.yaml

+"dataset_name": "high_humanities_history"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_high_humanities_history_egy"
+"task_alias": "high humanities history"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"

lm_eval/tasks/aradice/ArabicMMLU/EGY/AraDiCE_ArabicMMLU_high_humanities_islamic-studies.yaml

+"dataset_name": "high_humanities_islamic-studies"
+"description": ""
+"fewshot_split": !!null "null"
+"include": "_default_template_yaml"
+"tag": "AraDiCE_ArabicMMLU_humanities_egy"
+"task": "AraDiCE_ArabicMMLU_high_humanities_islamic-studies_egy"
+"task_alias": "high humanities islamic-studies"
+"test_split": "test"
+"training_split": !!null "null"
+"validation_split": !!null "null"