Merge branch 'EleutherAI:main' into main

lm_eval/tasks/agieval/README.md

@@ -75,7 +75,7 @@ Please make sure to cite all the individual datasets in your paper when you use
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 
@@ -89,6 +89,10 @@ Please make sure to cite all the individual datasets in your paper when you use
 
 - `agieval_nous`: Evaluates a specific subset of AGIEval tasks (multiple-choice and english-only), namely those in https://github.com/teknium1/LLM-Benchmark-Logs/blob/main/benchmark-logs/Mistral-7B-Base.md
 
+#### Tags
+
+None.
+
 #### Tasks
 
 - `agieval_aqua_rat`

lm_eval/tasks/agieval/agieval.yaml

+group: agieval
+task:
+  - agieval_gaokao_biology
+  - agieval_gaokao_chemistry
+  - agieval_gaokao_chinese
+  - agieval_gaokao_geography
+  - agieval_gaokao_history
+  - agieval_gaokao_mathcloze
+  - agieval_gaokao_mathqa
+  - agieval_gaokao_physics
+  - agieval_jec_qa_ca
+  - agieval_jec_qa_kd
+  - agieval_logiqa_zh
+  - agieval_aqua_rat
+  - agieval_gaokao_english
+  - agieval_logiqa_en
+  - agieval_lsat_ar
+  - agieval_lsat_lr
+  - agieval_lsat_rc
+  - agieval_math
+  - agieval_sat_en_without_passage
+  - agieval_sat_en
+  - agieval_sat_math
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/agieval/agieval_cn.yaml

+group: agieval_cn
+task:
+  - agieval_gaokao_biology
+  - agieval_gaokao_chemistry
+  - agieval_gaokao_chinese
+  - agieval_gaokao_geography
+  - agieval_gaokao_history
+  - agieval_gaokao_mathcloze
+  - agieval_gaokao_mathqa
+  - agieval_gaokao_physics
+  - agieval_jec_qa_ca
+  - agieval_jec_qa_kd
+  - agieval_logiqa_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/agieval/agieval_en.yaml

+group: agieval_en
+task:
+  - agieval_aqua_rat
+  - agieval_gaokao_english # categorizing as EN because the AGIEval codebase lists this as in `english_qa_tasks`
+  - agieval_logiqa_en
+  - agieval_lsat_ar
+  - agieval_lsat_lr
+  - agieval_lsat_rc
+  - agieval_math
+  - agieval_sat_en_without_passage
+  - agieval_sat_en
+  - agieval_sat_math
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/agieval/agieval_nous.yaml

+group: agieval_nous
+task:
+  - agieval_aqua_rat
+  - agieval_logiqa_en
+  - agieval_lsat_ar
+  - agieval_lsat_lr
+  - agieval_lsat_rc
+  - agieval_sat_en_without_passage
+  - agieval_sat_en
+  - agieval_sat_math
+aggregate_metric_list:
+  - metric: acc_norm
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/agieval/aqua-rat.yaml

-group:
-  - agieval
-  - agieval_en
-  - agieval_nous
 task: agieval_aqua_rat
 dataset_path: hails/agieval-aqua-rat
 dataset_name: null

lm_eval/tasks/agieval/gaokao-biology.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_biology
 dataset_path: hails/agieval-gaokao-biology

lm_eval/tasks/agieval/gaokao-chemistry.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_chemistry
 dataset_path: hails/agieval-gaokao-chemistry

lm_eval/tasks/agieval/gaokao-chinese.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_chinese
 dataset_path: hails/agieval-gaokao-chinese

lm_eval/tasks/agieval/gaokao-english.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_en # categorizing as EN because the AGIEval codebase lists this as in `english_qa_tasks`
 task: agieval_gaokao_english
 dataset_path: hails/agieval-gaokao-english

lm_eval/tasks/agieval/gaokao-geography.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_geography
 dataset_path: hails/agieval-gaokao-geography

lm_eval/tasks/agieval/gaokao-history.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_history
 dataset_path: hails/agieval-gaokao-history

lm_eval/tasks/agieval/gaokao-mathcloze.yaml

-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_mathcloze
 dataset_path: hails/agieval-gaokao-mathcloze
 dataset_name: null

lm_eval/tasks/agieval/gaokao-mathqa.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_mathqa
 dataset_path: hails/agieval-gaokao-mathqa

lm_eval/tasks/agieval/gaokao-physics.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_gaokao_physics
 dataset_path: hails/agieval-gaokao-physics

lm_eval/tasks/agieval/jec-qa-ca.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_jec_qa_ca
 dataset_path: hails/agieval-jec-qa-ca

lm_eval/tasks/agieval/jec-qa-kd.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_jec_qa_kd
 dataset_path: hails/agieval-jec-qa-kd

lm_eval/tasks/agieval/logiqa-en.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_logiqa_en
 dataset_path: hails/agieval-logiqa-en

lm_eval/tasks/agieval/logiqa-zh.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_cn
 task: agieval_logiqa_zh
 dataset_path: hails/agieval-logiqa-zh

lm_eval/tasks/agieval/lsat-ar.yaml

 include: aqua-rat.yaml
-group:
-  - agieval
-  - agieval_nous
-  - agieval_en
 task: agieval_lsat_ar
 dataset_path: hails/agieval-lsat-ar