Add Open Arabic LLM Leaderboard Benchmarks (Full and Light Version) (#2232)

* arabic leaferboard yaml file is added

* arabic toxigen is implemented

* Dataset library is imported

* arabic sciq is added

* util file of arabic toxigen is updated

* arabic race is added

* arabic piqa is implemented

* arabic open qa is added

* arabic copa is implemented

* arabic boolq ia added

* arabic arc easy is added

* arabic arc challenge is added

* arabic exams benchmark is implemented

* arabic hellaswag is added

* arabic leaderboard yaml file metrics are updated

* arabic mmlu benchmarks are added

* arabic mmlu group yaml file is updated

* alghafa benchmarks are added

* acva benchmarks are added

* acva utils.py is updated

* light version of arabic leaderboard benchmarks are added

* bugs fixed

* bug fixed

* bug fixed

* bug fixed

* bug fixed

* bug fixed

* library import bug is fixed

* doc to target updated

* bash file is deleted

* results folder is deleted

* leaderboard groups are added

* full arabic leaderboard groups are added, plus some bug fixes to the light version

* Create README.md

README.md for arabic_leaderboard_complete

* Create README.md

README.md for arabic_leaderboard_light

* Delete lm_eval/tasks/arabic_leaderboard directory

* Update README.md

* Update README.md

adding the Arabic leaderboards to the library

* Update README.md

10% of the training set

* Update README.md

10% of the training set

* revert .gitignore to prev version

* Update lm_eval/tasks/README.md
Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com>

* updated main README.md

* Update lm_eval/tasks/README.md

* specify machine translated benchmarks (complete)

* specify machine translated benchmarks (light version)

* add alghafa to the related task names (complete and light)

* add 'acva' to the related task names (complete and light)

* add 'arabic_leaderboard' to all the groups (complete and light)

* all dataset - not a random sample

* added more accurate details to the readme file

* added mt_mmlu from okapi

* Update lm_eval/tasks/README.md
Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com>

* Update lm_eval/tasks/README.md
Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com>

* updated mt_mmlu readme

* renaming 'alghafa' full and light

* renaming 'arabic_mmlu' light and full

* renaming 'acva' full and light

* update readme and standardize dir/file names

* running pre-commit

---------
Co-authored-by: shahrzads <sayehban@ualberta.ca>
Co-authored-by: shahrzads <56282669+shahrzads@users.noreply.github.com>
Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com>

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py

+import datasets
+import numpy as np
+
+
+def process_docs(dataset: datasets.Dataset):
+    def _process_doc(doc):
+        question = doc["query"]
+        answer_index = int(doc["label"])
+        # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label'
+        choices_keys = [
+            key for key in doc.keys() if key not in ["query", "label", "__few_shots"]
+        ]
+        choices = [doc[key] for key in choices_keys]
+
+        instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n"
+        query = f"{instruction}السؤال: {question}\n"
+        for index, choice in enumerate(choices):
+            query += f"{index}) {choice}\n"
+        query += "الإجابة:"
+
+        return {"query": query, "choices": choices, "gold": answer_index}
+
+    return dataset.map(_process_doc)

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_exams_light.yaml

+task: arabic_exams_light
+dataset_path: arcee-globe/Arabic_EXAMS-10percent
+dataset_name: default
+output_type: multiple_choice
+training_split: null
+validation_split: validation
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: validation
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_leaderboard_arabic_exams_light.yaml

+group: arabic_leaderboard_arabic_exams_light
+task:
+  - arabic_exams_light
+
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+  - metric: acc_norm
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py

+import datasets
+import numpy as np
+
+
+# fmt: off
+LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"]
+# fmt: on
+
+
+# fmt: off
+LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]
+# fmt: on
+
+
+def process_docs(dataset: datasets.Dataset):
+    def _process_doc(doc):
+        topic = doc["subject"]
+        question = doc["question"]
+        choices = [doc["A"], doc["B"], doc["C"], doc["D"]]
+        choices_formatted = [
+            f" {LETTER_INDICES_AR[i]}) {choice}\n" for i, choice in enumerate(choices)
+        ]
+        answer = doc["answer"]
+        answer_index = LETTER_INDICES.index(answer)
+
+        instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n"
+        query = f"{instruction}السؤال: {question}\n"
+        query += "\n".join(choices_formatted)
+        query += "\nالإجابة:"
+
+        return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": answer_index}
+
+    return dataset.map(_process_doc)

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml

+task: arabic_leaderboard_arabic_mmlu_abstract_algebra_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: abstract_algebra
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_anatomy_light.yaml

+task: arabic_leaderboard_arabic_mmlu_anatomy_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: anatomy
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_astronomy_light.yaml

+task: arabic_leaderboard_arabic_mmlu_astronomy_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: astronomy
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml

+task: arabic_leaderboard_arabic_mmlu_business_ethics_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: business_ethics
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml

+task: arabic_leaderboard_arabic_mmlu_clinical_knowledge_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: clinical_knowledge
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_biology_light.yaml

+task: arabic_leaderboard_arabic_mmlu_college_biology_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: college_biology
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml

+task: arabic_leaderboard_arabic_mmlu_college_chemistry_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: college_chemistry
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml

+task: arabic_leaderboard_arabic_mmlu_college_computer_science_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: college_computer_science
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml

+task: arabic_leaderboard_arabic_mmlu_college_mathematics_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: college_mathematics
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml

+task: arabic_leaderboard_arabic_mmlu_college_medicine_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: college_medicine
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_physics_light.yaml

+task: arabic_leaderboard_arabic_mmlu_college_physics_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: college_physics
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_computer_security_light.yaml

+task: arabic_leaderboard_arabic_mmlu_computer_security_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: computer_security
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml

+task: arabic_leaderboard_arabic_mmlu_conceptual_physics_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: conceptual_physics
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_econometrics_light.yaml

+task: arabic_leaderboard_arabic_mmlu_econometrics_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: econometrics
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml

+task: arabic_leaderboard_arabic_mmlu_electrical_engineering_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: electrical_engineering
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml

+task: arabic_leaderboard_arabic_mmlu_elementary_mathematics_light
+dataset_path: arcee-globe/Arabic_MMLU-10percent
+dataset_name: elementary_mathematics
+output_type: multiple_choice
+training_split: null
+validation_split: dev
+test_split: test
+process_docs: !function utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0