Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into multiprompt

lm_eval/tasks/glue/cola/default.yaml

-group: glue
+tag: glue
 task: cola
 dataset_path: glue
 dataset_name: cola

lm_eval/tasks/glue/mnli/default.yaml

-group: glue
+tag: glue
 task: mnli
 dataset_path: glue
 dataset_name: mnli

lm_eval/tasks/glue/mrpc/default.yaml

-group: glue
+tag: glue
 task: mrpc
 dataset_path: glue
 dataset_name: mrpc

lm_eval/tasks/glue/qnli/default.yaml

-group: glue
+tag: glue
 task: qnli
 dataset_path: glue
 dataset_name: qnli

lm_eval/tasks/glue/qqp/default.yaml

-group: glue
+tag: glue
 task: qqp
 dataset_path: glue
 dataset_name: qqp

lm_eval/tasks/glue/rte/default.yaml

-group: glue
+tag: glue
 task: rte
 dataset_path: glue
 dataset_name: rte

lm_eval/tasks/glue/sst2/default.yaml

-group: glue
+tag: glue
 task: sst2
 dataset_path: glue
 dataset_name: sst2

lm_eval/tasks/glue/wnli/default.yaml

-group: glue
+tag: glue
 task: wnli
 dataset_path: glue
 dataset_name: wnli

lm_eval/tasks/gpqa/README.md

@@ -25,11 +25,15 @@ Homepage: `https://github.com/idavidrein/gpqa/tree/main`
 
 This dataset is gated, so you will have to accept the terms of use at https://huggingface.co/datasets/Idavidrein/gpqa and login via `huggingface-cli login` using your HF Hub token before running this task.
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 
-* `gpqa`
+None
+
+#### Tags
+
+* `gpqa`: runs all GPQA variants.
 
 #### Tasks
 

lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml

 dataset_path: Idavidrein/gpqa
-group: gpqa
+tag: gpqa
 output_type: generate_until
 process_docs: !function utils.process_docs
 training_split: train

lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml

 dataset_path: Idavidrein/gpqa
-group: gpqa
+tag: gpqa
 output_type: generate_until
 process_docs: !function utils.process_docs
 training_split: train

lm_eval/tasks/gpqa/generative/_gpqa_generative_n_shot_yaml

 dataset_path: Idavidrein/gpqa
-group: gpqa
+tag: gpqa
 output_type: generate_until
 process_docs: !function utils.process_docs
 training_split: train

lm_eval/tasks/gpqa/n_shot/_gpqa_n_shot_yaml

 dataset_path: Idavidrein/gpqa
-group: gpqa
+tag: gpqa
 output_type: multiple_choice
 process_docs: !function utils.process_docs
 training_split: train

lm_eval/tasks/gpqa/zeroshot/_gpqa_zeroshot_yaml

 dataset_path: Idavidrein/gpqa
-group: gpqa
+tag: gpqa
 output_type: multiple_choice
 process_docs: !function utils.process_docs
 training_split: train

lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml

 include: gsm8k-cot.yaml
-group:
+tag:
   - chain_of_thought
   - self_consistency
 task: gsm8k_cot_self_consistency

lm_eval/tasks/gsm8k/gsm8k-cot-zeroshot.yaml

-group:
+tag:
   - math_word_problems
 task: gsm8k_cot_zeroshot
 dataset_path: gsm8k

lm_eval/tasks/gsm8k/gsm8k-cot.yaml

@@ -61,7 +61,7 @@ generation_kwargs:
   - 'Q:'
   - </s>
   - <|im_end|>
-group:
+tag:
 - chain_of_thought
 metadata:
   version: 3.0

lm_eval/tasks/gsm8k/gsm8k.yaml

-group:
+tag:
   - math_word_problems
 task: gsm8k
 dataset_path: gsm8k

lm_eval/tasks/haerae/_default_haerae_yaml

-group: haerae
 dataset_path: HAERAE-HUB/HAE_RAE_BENCH
 test_split: test
 fewshot_split: test

lm_eval/tasks/haerae/_haerae.yaml

+group: haerae
+task:
+  - haerae_gk
+  - haerae_hi
+  - haerae_lw
+  - haerae_rw
+  - haerae_sn
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+  - metric: acc_norm
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0