Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into multiprompt

lm_eval/tasks/inverse_scaling/inverse_scaling_redefine_math.yaml

+include: _inverse_scaling_mc_yaml
+task: inverse_scaling_redefine_math
+dataset_path: inverse-scaling/redefine-math

lm_eval/tasks/inverse_scaling/inverse_scaling_repetitive_algebra.yaml

+include: _inverse_scaling_mc_yaml
+task: inverse_scaling_repetitive_algebra
+dataset_path: Albertmade/repetitive-algebra

lm_eval/tasks/inverse_scaling/inverse_scaling_sig_figs.yaml

+include: _inverse_scaling_mc_yaml
+task: inverse_scaling_sig_figs
+dataset_path: Albertmade/sig-figs

lm_eval/tasks/inverse_scaling/inverse_scaling_winobias_antistereotype.yaml

+group:
+  - inverse_scaling_mc
+task: inverse_scaling_winobias_antistereotype
+dataset_path: mathemakitten/winobias_antistereotype_test_v5
+output_type: multiple_choice
+test_split: test
+doc_to_text: text
+doc_to_choice: classes
+doc_to_target: target
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+dataset_kwargs:
+  trust_remote_code: true
+metadata:
+  version: 0

lm_eval/tasks/kmmlu/cot_hard/_cot_kmmlu_yaml

-group:
+tag:
     - kmmlu
     - kmmlu_hard_cot
 dataset_path: HAERAE-HUB/KMMLU-HARD

lm_eval/tasks/kmmlu/direct/_direct_kmmlu_yaml

-group:
+tag:
     - kmmlu
     - kmmlu_direct
 dataset_path: HAERAE-HUB/KMMLU

lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml

-group:
+tag:
     - kmmlu
     - kmmlu_hard_direct
 dataset_path: HAERAE-HUB/KMMLU-HARD

lm_eval/tasks/kmmlu/hard/_hard_kmmlu_yaml

-group:
+tag:
     - kmmlu
     - kmmlu_hard
 dataset_path: HAERAE-HUB/KMMLU-HARD

lm_eval/tasks/kobest/kobest_boolq.yaml

-group:
+tag:
   - kobest
 task: kobest_boolq
 dataset_path: skt/kobest_v1

lm_eval/tasks/kobest/kobest_copa.yaml

-group:
+tag:
   - kobest
 task: kobest_copa
 dataset_path: skt/kobest_v1

lm_eval/tasks/kobest/kobest_hellaswag.yaml

-group:
+tag:
   - kobest
 task: kobest_hellaswag
 dataset_path: skt/kobest_v1

lm_eval/tasks/kobest/kobest_sentineg.yaml

-group:
+tag:
   - kobest
 task: kobest_sentineg
 dataset_path: skt/kobest_v1

lm_eval/tasks/kobest/kobest_wic.yaml

-group:
+tag:
   - kobest
 task: kobest_wic
 dataset_path: skt/kobest_v1

lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml

+group: kormedmcqa
+task:
+  - kormedmcqa_doctor
+  - kormedmcqa_nurse
+  - kormedmcqa_pharm
+aggregate_metric_list:
+  - metric: exact_match
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/kormedmcqa/kormedmcqa_doctor.yaml

-group: kormedmcqa
 task : kormedmcqa_doctor
 dataset_path : sean0042/KorMedMCQA
 dataset_name : doctor

lm_eval/tasks/kormedmcqa/kormedmcqa_nurse.yaml

-group: kormedmcqa
 task : kormedmcqa_nurse
 dataset_path : sean0042/KorMedMCQA
 dataset_name : nurse

lm_eval/tasks/kormedmcqa/kormedmcqa_pharm.yaml

-group: kormedmcqa
 task : kormedmcqa_pharm
 dataset_path : sean0042/KorMedMCQA
 dataset_name : pharm

lm_eval/tasks/lambada/lambada_openai.yaml

-group:
+tag:
   - lambada
 task: lambada_openai
 dataset_path: EleutherAI/lambada_openai

lm_eval/tasks/lambada/lambada_standard.yaml

-group:
+tag:
   - lambada
 task: lambada_standard
 dataset_path: lambada

lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml

-group:
+tag:
   - lambada_cloze
 task: lambada_openai_cloze_yaml
 dataset_path: EleutherAI/lambada_openai