Merge branch 'main' into multimodal-prototyping

lm_eval/tasks/agieval/agieval_cn.yaml

+group: agieval_cn
+task:
+  - agieval_gaokao_biology
+  - agieval_gaokao_chemistry
+  - agieval_gaokao_chinese
+  - agieval_gaokao_geography
+  - agieval_gaokao_history
+  - agieval_gaokao_mathcloze
+  - agieval_gaokao_mathqa
+  - agieval_gaokao_physics
+  - agieval_jec_qa_ca
+  - agieval_jec_qa_kd
+  - agieval_logiqa_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/agieval/agieval_en.yaml

+group: agieval_en
+task:
+  - agieval_aqua_rat
+  - agieval_gaokao_english # categorizing as EN because the AGIEval codebase lists this as in `english_qa_tasks`
+  - agieval_logiqa_en
+  - agieval_lsat_ar
+  - agieval_lsat_lr
+  - agieval_lsat_rc
+  - agieval_math
+  - agieval_sat_en_without_passage
+  - agieval_sat_en
+  - agieval_sat_math
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/agieval/agieval_nous.yaml

+group: agieval_nous
+task:
+  - agieval_aqua_rat
+  - agieval_logiqa_en
+  - agieval_lsat_ar
+  - agieval_lsat_lr
+  - agieval_lsat_rc
+  - agieval_sat_en_without_passage
+  - agieval_sat_en
+  - agieval_sat_math
+aggregate_metric_list:
+  - metric: acc_norm
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/agieval/aqua-rat.yaml

-group:
-  - agieval
-  - agieval_en
-  - agieval_nous
 task: agieval_aqua_rat
 dataset_path: hails/agieval-aqua-rat
 dataset_name: null

lm_eval/tasks/agieval/gaokao-biology.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_biology
 dataset_path: hails/agieval-gaokao-biology

lm_eval/tasks/agieval/gaokao-chemistry.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_chemistry
 dataset_path: hails/agieval-gaokao-chemistry

lm_eval/tasks/agieval/gaokao-chinese.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_chinese
 dataset_path: hails/agieval-gaokao-chinese

lm_eval/tasks/agieval/gaokao-english.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_en # categorizing as EN because the AGIEval codebase lists this as in `english_qa_tasks`
 task: agieval_gaokao_english
 dataset_path: hails/agieval-gaokao-english

lm_eval/tasks/agieval/gaokao-geography.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_geography
 dataset_path: hails/agieval-gaokao-geography

lm_eval/tasks/agieval/gaokao-history.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_history
 dataset_path: hails/agieval-gaokao-history

lm_eval/tasks/agieval/gaokao-mathcloze.yaml

-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_mathcloze
 dataset_path: hails/agieval-gaokao-mathcloze
 dataset_name: null

lm_eval/tasks/agieval/gaokao-mathqa.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_mathqa
 dataset_path: hails/agieval-gaokao-mathqa

lm_eval/tasks/agieval/gaokao-physics.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_physics
 dataset_path: hails/agieval-gaokao-physics

lm_eval/tasks/agieval/jec-qa-ca.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_jec_qa_ca
 dataset_path: hails/agieval-jec-qa-ca

lm_eval/tasks/agieval/jec-qa-kd.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_jec_qa_kd
 dataset_path: hails/agieval-jec-qa-kd

lm_eval/tasks/agieval/logiqa-en.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_logiqa_en
 dataset_path: hails/agieval-logiqa-en

lm_eval/tasks/agieval/logiqa-zh.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_logiqa_zh
 dataset_path: hails/agieval-logiqa-zh

lm_eval/tasks/agieval/lsat-ar.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_lsat_ar
 dataset_path: hails/agieval-lsat-ar

lm_eval/tasks/agieval/lsat-lr.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_lsat_lr
 dataset_path: hails/agieval-lsat-lr

lm_eval/tasks/agieval/lsat-rc.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_lsat_rc
 dataset_path: hails/agieval-lsat-rc