Merge branch 'main' into mathvista

lm_eval/models/vllm_causallms.py

@@ -76,9 +76,9 @@ class VLLM(TemplateLM):
             )
 
         assert "cuda" in device or device is None, "vLLM only supports CUDA"
-        assert (
-            max_length is None or max_model_len is None
-        ), "Either max_length or max_model_len may be provided, but not both"
+        assert max_length is None or max_model_len is None, (
+            "Either max_length or max_model_len may be provided, but not both"
+        )
 
         self._max_length = max_model_len if max_model_len is not None else max_length
         self.tensor_parallel_size = int(tensor_parallel_size)
@@ -142,9 +142,9 @@ class VLLM(TemplateLM):
         self._max_gen_toks = max_gen_toks
 
         if lora_local_path is not None:
-            assert parse_version(version("vllm")) > parse_version(
-                "0.3.0"
-            ), "lora adapters only compatible with vllm > v0.3.0."
+            assert parse_version(version("vllm")) > parse_version("0.3.0"), (
+                "lora adapters only compatible with vllm > v0.3.0."
+            )
             self.lora_request = LoRARequest("finetuned", 1, lora_local_path)
         else:
             self.lora_request = None

lm_eval/models/vllm_vlms.py

@@ -271,7 +271,9 @@ class VLLM_VLM(VLLM):
                 left_truncate_len=max_ctx_len,
             )
 
-            cont = self._model_generate(inputs, stop=until, generate=True, **kwargs)
+            cont = self._model_generate(
+                inputs, stop=until, generate=True, max_tokens=max_gen_toks, **kwargs
+            )
 
             for output, context in zip(cont, contexts):
                 generated_text = output.outputs[0].text

lm_eval/tasks/arabicmmlu/_arabicmmlu.yaml

@@ -9,4 +9,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_humanities.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_language.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_other.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_social_science.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_arabicmmlu_stem.yaml

@@ -6,4 +6,4 @@ aggregate_metric_list:
   - metric: acc
     weight_by_size: True
 metadata:
-  version: 0
+  version: 1

lm_eval/tasks/arabicmmlu/_default_arabicmmlu_template_yaml

-dataset_path: yazeed7/ArabicMMLU
+dataset_path: MBZUAI/ArabicMMLU
 test_split: test
 fewshot_split: dev
 fewshot_config:
@@ -12,4 +12,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/arabicmmlu/_generate_configs.py

@@ -14,46 +14,46 @@ eval_logger = logging.getLogger("lm-eval")
 
 
 SUBJECTS = {
-    "Driving Test": "other",
-    "High Geography": "social_science",
-    "High History": "humanities",
     "Islamic Studies": "humanities",
-    "Univ Accounting": "social_science",
-    "Primary General Knowledge": "other",
-    "Univ Political Science": "social_science",
-    "Primary Math": "stem",
-    "Middle General Knowledge": "other",
-    "High Biology": "stem",
-    "Primary Natural Science": "stem",
-    "High Economics": "social_science",
-    "Middle Natural Science": "stem",
-    "Middle Geography": "social_science",
-    "Primary Social Science": "social_science",
-    "Middle Computer Science": "stem",
-    "Middle Islamic Studies": "humanities",
-    "Primary Computer Science": "stem",
-    "High Physics": "stem",
-    "Middle Social Science": "social_science",
-    "Middle Civics": "social_science",
-    "High Computer Science": "stem",
+    "Driving Test": "other",
+    "Natural Science (Middle School)": "stem",
+    "Natural Science (Primary School)": "stem",
+    "History (Primary School)": "humanities",
+    "History (Middle School)": "humanities",
+    "History (High School)": "humanities",
     "General Knowledge": "other",
-    "High Civics": "social_science",
-    "Prof Law": "humanities",
-    "High Islamic Studies": "humanities",
-    "Primary Arabic Language": "language",
-    "High Arabic Language": "language",
-    "Arabic Language (Grammar)": "language",
-    "Primary History": "humanities",
-    "Middle History": "humanities",
-    "Univ Economics": "social_science",
+    "General Knowledge (Primary School)": "other",
+    "General Knowledge (Middle School)": "other",
+    "Law (Professional)": "humanities",
+    "Physics (High School)": "stem",
+    "Social Science (Middle School)": "social_science",
+    "Social Science (Primary School)": "social_science",
+    "Management (University)": "other",
+    "Arabic Language (Primary School)": "language",
+    "Arabic Language (Middle School)": "language",
+    "Arabic Language (High School)": "language",
+    "Political Science (University)": "social_science",
+    "Philosophy (High School)": "humanities",
+    "Accounting (University)": "social_science",
+    "Computer Science (University)": "stem",
+    "Computer Science (Middle School)": "stem",
+    "Computer Science (Primary School)": "stem",
+    "Computer Science (High School)": "stem",
+    "Geography (Primary School)": "social_science",
+    "Geography (Middle School)": "social_science",
+    "Geography (High School)": "social_science",
+    "Math (Primary School)": "stem",
+    "Biology (High School)": "stem",
+    "Economics (University)": "social_science",
+    "Economics (Middle School)": "social_science",
+    "Economics (High School)": "social_science",
     "Arabic Language (General)": "language",
-    "Univ Computer Science": "stem",
-    "Primary Islamic Studies": "humanities",
-    "Primary Geography": "social_science",
-    "High Philosophy": "humanities",
-    "Middle Arabic Language": "language",
-    "Middle Economics": "social_science",
-    "Univ Management": "other",
+    "Arabic Language (Grammar)": "language",
+    "Islamic Studies (High School)": "humanities",
+    "Islamic Studies (Middle School)": "humanities",
+    "Islamic Studies (Primary School)": "humanities",
+    "Civics (Middle School)": "social_science",
+    "Civics (High School)": "social_science",
 }
 
 
@@ -69,8 +69,9 @@ if __name__ == "__main__":
 
     # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs.
     base_yaml_name = os.path.split(args.base_yaml_path)[-1]
-    with open(args.base_yaml_path, encoding="utf-8") as f:
-        base_yaml = yaml.full_load(f)
+
+    # with open(args.base_yaml_path, encoding="utf-8") as f:
+    #     base_yaml = yaml.full_load(f)
 
     ALL_CATEGORIES = []
     for subject, category in tqdm(SUBJECTS.items()):
@@ -81,8 +82,8 @@ if __name__ == "__main__":
 
         yaml_dict = {
             "include": base_yaml_name,
-            "tag": f"arabicmmlu_{category}",
-            "task": f"arabicmmlu_{subject.lower().replace(' ', '_')}",
+            "tag": f"arabicmmlu_{category}_tasks",
+            "task": f"arabicmmlu_{subject.lower().replace(' ', '_').replace('(', '').replace(')', '')}",
             "task_alias": subject,
             "dataset_name": subject,
             # "description": description,

lm_eval/tasks/arabicmmlu/arabicmmlu_middle_civics.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_accounting_university.yaml

-"dataset_name": "Middle Civics"
-"tag": "arabicmmlu_social_science_tasks"
+"dataset_name": "Accounting (University)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_middle_civics"
-"task_alias": "Middle Civics"
+"tag": "arabicmmlu_social_science_tasks"
+"task": "arabicmmlu_accounting_university"
+"task_alias": "Accounting (University)"

lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml

 "dataset_name": "Arabic Language (General)"
-"tag": "arabicmmlu_language_tasks"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_arabic_language_(general)"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_general"
 "task_alias": "Arabic Language (General)"

lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_grammar.yaml

 "dataset_name": "Arabic Language (Grammar)"
-"tag": "arabicmmlu_language_tasks"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_arabic_language_(grammar)"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_grammar"
 "task_alias": "Arabic Language (Grammar)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_arabic_language.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_high_school.yaml

-"dataset_name": "High Arabic Language"
-"tag": "arabicmmlu_language_tasks"
+"dataset_name": "Arabic Language (High School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_arabic_language"
-"task_alias": "High Arabic Language"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_high_school"
+"task_alias": "Arabic Language (High School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_middle_arabic_language.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_middle_school.yaml

-"dataset_name": "Middle Arabic Language"
-"tag": "arabicmmlu_language_tasks"
+"dataset_name": "Arabic Language (Middle School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_middle_arabic_language"
-"task_alias": "Middle Arabic Language"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_middle_school"
+"task_alias": "Arabic Language (Middle School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_primary_school.yaml

-"dataset_name": "Primary Arabic Language"
-"tag": "arabicmmlu_language_tasks"
+"dataset_name": "Arabic Language (Primary School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_primary_arabic_language"
-"task_alias": "Primary Arabic Language"
+"tag": "arabicmmlu_language_tasks"
+"task": "arabicmmlu_arabic_language_primary_school"
+"task_alias": "Arabic Language (Primary School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_physics.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_biology_high_school.yaml

-"dataset_name": "High Physics"
-"tag": "arabicmmlu_stem_tasks"
+"dataset_name": "Biology (High School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_physics"
-"task_alias": "High Physics"
+"tag": "arabicmmlu_stem_tasks"
+"task": "arabicmmlu_biology_high_school"
+"task_alias": "Biology (High School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_economics.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_civics_high_school.yaml

-"dataset_name": "High Economics"
-"tag": "arabicmmlu_social_science_tasks"
+"dataset_name": "Civics (High School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_economics"
-"task_alias": "High Economics"
+"tag": "arabicmmlu_social_science_tasks"
+"task": "arabicmmlu_civics_high_school"
+"task_alias": "Civics (High School)"

lm_eval/tasks/arabicmmlu/arabicmmlu_high_geography.yaml → lm_eval/tasks/arabicmmlu/arabicmmlu_civics_middle_school.yaml

-"dataset_name": "High Geography"
-"tag": "arabicmmlu_social_science_tasks"
+"dataset_name": "Civics (Middle School)"
 "include": "_default_arabicmmlu_template_yaml"
-"task": "arabicmmlu_high_geography"
-"task_alias": "High Geography"
+"tag": "arabicmmlu_social_science_tasks"
+"task": "arabicmmlu_civics_middle_school"
+"task_alias": "Civics (Middle School)"