Merge branch 'main' into metrics

# Conflicts:
#	tests/test_tasks.py

lm_eval/tasks/egymmlu/egymmlu_moral_scenarios.yaml

+"dataset_name": "moral_scenarios"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_humanities_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_moral_scenarios"
+"task_alias": "moral scenarios"

lm_eval/tasks/egymmlu/egymmlu_natural_science.yaml

+"dataset_name": "natural_science"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_stem_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_natural_science"
+"task_alias": "natural science"

lm_eval/tasks/egymmlu/egymmlu_nutrition.yaml

+"dataset_name": "nutrition"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_other_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_nutrition"
+"task_alias": "nutrition"

lm_eval/tasks/egymmlu/egymmlu_philosophy.yaml

+"dataset_name": "philosophy"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_humanities_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_philosophy"
+"task_alias": "philosophy"

lm_eval/tasks/egymmlu/egymmlu_philosophy_ar.yaml

+"dataset_name": "philosophy_ar"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_humanities_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_philosophy_ar"
+"task_alias": "philosophy ar"

lm_eval/tasks/egymmlu/egymmlu_physics.yaml

+"dataset_name": "physics"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_stem_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_physics"
+"task_alias": "physics"

lm_eval/tasks/egymmlu/egymmlu_political_science.yaml

+"dataset_name": "political_science"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_political_science"
+"task_alias": "political science"

lm_eval/tasks/egymmlu/egymmlu_professional_law.yaml

+"dataset_name": "professional_law"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_humanities_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_professional_law"
+"task_alias": "professional law"

lm_eval/tasks/egymmlu/egymmlu_professional_psychology.yaml

+"dataset_name": "professional_psychology"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_professional_psychology"
+"task_alias": "professional psychology"

lm_eval/tasks/egymmlu/egymmlu_public_relations.yaml

+"dataset_name": "public_relations"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_public_relations"
+"task_alias": "public relations"

lm_eval/tasks/egymmlu/egymmlu_security_studies.yaml

+"dataset_name": "security_studies"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_security_studies"
+"task_alias": "security studies"

lm_eval/tasks/egymmlu/egymmlu_social_science.yaml

+"dataset_name": "social_science"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_social_science"
+"task_alias": "social science"

lm_eval/tasks/egymmlu/egymmlu_sociology.yaml

+"dataset_name": "sociology"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_sociology"
+"task_alias": "sociology"

lm_eval/tasks/egymmlu/egymmlu_world_religions.yaml

+"dataset_name": "world_religions"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_humanities_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_world_religions"
+"task_alias": "world religions"

lm_eval/tasks/egymmlu/utils.py

+PROMPT = "ده سؤال متعدد الاختيار (مع إجابته) على {}\n\n{}\n{}\n الجواب:"
+
+
+alpha = ["A.", "B.", "C.", "D.", "E."]
+
+
+def doc_to_text(doc):
+    subject = doc["egy_subject"]  # subject_egyptian
+    question = (
+        doc["question"]
+        if doc["context"] == ""
+        else f"{doc['context']}\n\n{doc['question']}"
+    )
+
+    options = []
+    for i, opt in enumerate(doc["choices"]):
+        options.append(f"{alpha[i]} {opt}")
+
+    doc_text = PROMPT.format(subject, question, "\n".join(options))
+
+    return doc_text
+
+
+def doc_to_choice(doc):
+    return [alpha[i][0] for i in range(len(doc["choices"]))]

lm_eval/tasks/meddialog/utils.py

@@ -11,7 +11,9 @@ try:
 
 except (ModuleNotFoundError, ImportError):
     raise ModuleNotFoundError(
-        "Please install evaluation metrics via pip install evaluate and pip install bert-score",
+        "Please install evaluation metrics via pip install evaluate bert-score "
+        "rouge_score>=0.1.2 nltk absl-py "
+        "git+https://github.com/google-research/bleurt.git"
     )
 except Exception as e:
     raise RuntimeError(

lm_eval/tasks/mediqa_qa2019/utils.py

@@ -11,7 +11,9 @@ try:
 
 except (ModuleNotFoundError, ImportError):
     raise ModuleNotFoundError(
-        "Please install evaluation metrics via pip install evaluate and pip install bert-score",
+        "Please install evaluation metrics via pip install evaluate bert-score "
+        "rouge_score>=0.1.2 nltk absl-py "
+        "git+https://github.com/google-research/bleurt.git"
     )
 except Exception as e:
     raise RuntimeError(