add mathvista_mcq

lm_eval/tasks/mathvista/mathvista_mcq.yaml

+include: mathvista.yaml
+task: mathvista_mcq
+output_type: "multiple_choice"
+process_docs: !function utils.process_docs_mcq
+doc_to_choice: '{{ ["A", "B", "C", "D", "E", "F"][:choices.length] }}'
+doc_to_target: "{{choices.index(answer)}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/mathvista/utils.py

@@ -143,3 +143,10 @@ def process_results(doc: dict, results: list[str]):
     )
     res = safe_equal(normalized_extraction, answer)
     return {"acc": 1.0} if res else {"acc": 0.0}
+
+
+### MathVista MCQ ###
+
+
+def process_docs_mcq(dataset):
+    return dataset.filter(lambda x: x["question_type"] == "multi_choice")