add ai2d

6dc55fb3 · Baber · 1a28d18f · 6dc55fb3
Commit 6dc55fb3 authored Sep 19, 2024 by Baber
Hide whitespace changes
Inline Side-by-side

Showing with 16 additions and 0 deletions

lm_eval/tasks/ai2d/ai2d.yaml lm_eval/tasks/ai2d/ai2d.yaml +16 -0

No files found.
--- a/lm_eval/tasks/ai2d/ai2d.yaml
+++ b/lm_eval/tasks/ai2d/ai2d.yaml
+task: ai2d
+dataset_path: lmms-lab/ai2d
+output_type: multiple_choice
+test_split: test
+doc_to_text: "<image> Question: {{question}}\nAnswer:"
+doc_to_target: "answer"
+doc_to_choice: "{{options}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0