ai2d.yaml

dataset_path: lmms-lab/ai2d
test_split: test
output_type: generate_until
doc_to_image: !function utils.doc_to_image
doc_to_text: "<image>{{question | capitalize}}
{{% for option in options %}}
{{loop.index | chr(64)}}. {{option | capitalize}}
{{% endfor %}}"
doc_to_target: "{{options[answer]}}"
generation_kwargs:
  until: []
  temperature: 0.0
  do_sample: false
  max_gen_toks: 512
metric_list:
  - metric: acc
    aggregation: mean
    higher_is_better: true
metadata:
  version: 0.0