multimedqa.yaml

group: multimedqa
task:
  - pubmedqa
  - medmcqa
  - medqa_4options
  - task: mmlu_anatomy
    task_alias: "anatomy (mmlu)"
  - task: mmlu_clinical_knowledge
    task_alias: "clinical_knowledge (mmlu)"
  - task: mmlu_college_medicine
    task_alias: "college_medicine (mmlu)"
  - task: mmlu_medical_genetics
    task_alias: "medical_genetics (mmlu)"
  - task: mmlu_professional_medicine
    task_alias: "professional_medicine (mmlu)"
  - task: mmlu_college_biology
    task_alias: "college_biology (mmlu)"
aggregate_metric_list:
  - metric: acc
    aggregation: mean
    weight_by_size: True