add task yamls

lm_eval/tasks/mmmu/_mmmu_mc_yaml

+include: _template_yaml
+process_docs: !function utils.process_multiple_choice

lm_eval/tasks/mmmu/_mmmu_open_yaml

+include: _template_yaml
+process_docs: !utils.process_open_choice
+dataset_name: Electronics
\ No newline at end of file

lm_eval/tasks/mmmu/mmmu.yaml → lm_eval/tasks/mmmu/_template_yaml

-dataset_path: lmms-lab/MMMU
+dataset_path: MMMU/MMMU
 task: "mmmu_val"
 validation_split: validation
 output_type: generate_until
-input_type: text_image
 doc_to_visual: !function utils.mmmu_doc_to_visual
 doc_to_text: !function utils.mmmu_doc_to_text
 doc_to_target: "answer"
 # The return value of process_results will be used by metrics
-process_results: !function utils.mmmu_process_results
+# process_results: !function utils.mmmu_process_results
 # Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
 generation_kwargs:
   until:
@@ -19,6 +18,7 @@ generation_kwargs:
   repetition_penalty: 1.0
   image_aspect_ratio: original
 metric_list:
-  - metric: mmmu_acc
-    aggregation: !function utils.mmmu_aggregate_results
-    higher_is_better: true
+  - metric: acc
+  # - metric: mmmu_acc
+  #   aggregation: !function utils.mmmu_aggregate_results
\ No newline at end of file
+  #   higher_is_better: true

lm_eval/tasks/mmmu/mmmu_electronics.yaml

+task: mmmu_electronics
+include: _mmmu_mc_yaml
+dataset_name: Electronics
\ No newline at end of file

lm_eval/tasks/mmmu/mmmu_yaml

+group: mmmu
+task:
+  - mmmu_mc
+  - mmmu_open
\ No newline at end of file

lm_eval/tasks/mmmu/utils.py

@@ -475,3 +475,9 @@ def get_multi_choice_info(options):
         all_choices.append(chr(ord(start_chr) + i))
 
     return index2ans, all_choices
+
+def process_multiple_choice(dataset):
+    return dataset.filter(lambda example: example["question_type"] == "multiple-choice")
+
+def process_open_choice(dataset):
+    return dataset.filter(lambda example: example["question_type"] == "open")
\ No newline at end of file