Merge branch 'main' of https://github.com/EleutherAI/lm-evaluation-harness into mela

lm_eval/tasks/french_bench/french_bench_orangesum_abstract.yaml

 include: "_default_template_yaml"
-group:
+tag:
   - french_bench
   - french_bench_gen
 description: "Résume l'article en une phrase."

lm_eval/tasks/french_bench/french_bench_orangesum_title.yaml

 include: "_default_template_yaml"
-group:
+tag:
   - french_bench
   - french_bench_extra
 description: "Trouve le titre de l'article."

lm_eval/tasks/french_bench/french_bench_reading_comp.yaml

 include: "_default_template_yaml"
-group:
+tag:
   - french_bench
   - french_bench_extra
 # description: "Répond au mieux en complétant la question avec une des réponses proposées."

lm_eval/tasks/french_bench/french_bench_topic_based_nli.yaml

 include: "_default_template_yaml"
-group:
+tag:
   - french_bench
   - french_bench_extra
 description: "A propos du thème spécifié, l'avis client est il positif, négatif, ou neutre ?"

lm_eval/tasks/french_bench/french_bench_trivia.yaml

 include: "_default_template_yaml"
-group:
+tag:
   - french_bench
   - french_bench_gen
 task: french_bench_trivia

lm_eval/tasks/french_bench/french_bench_vocab.yaml

 include: "_default_template_yaml"
-group:
+tag:
   - french_bench
   - french_bench_mc
 # description: "Répond au mieux en complétant la question avec une des réponses proposées."

lm_eval/tasks/french_bench/french_bench_wikitext_fr.yaml

-group:
+tag:
   - french_bench_perplexity
 task: french_bench_wikitext_fr
 dataset_path: asi/wikitext_fr

lm_eval/tasks/french_bench/french_bench_xnli.yaml

 include: "_default_template_yaml"
-group:
+tag:
   - french_bench
   - french_bench_extra
 description: "La prémisse et l'hypothèse sont elles en accord, neutres en elles, ou en contradiction ?"

lm_eval/tasks/glue/README.md

@@ -41,10 +41,14 @@ Homepage: https://gluebenchmark.com/
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 
+None.
+
+#### Tags
+
 * `glue`: Run all Glue subtasks.
 
 #### Tasks

lm_eval/tasks/glue/cola/default.yaml

-group: glue
+tag: glue
 task: cola
 dataset_path: glue
 dataset_name: cola

lm_eval/tasks/glue/mnli/default.yaml

-group: glue
+tag: glue
 task: mnli
 dataset_path: glue
 dataset_name: mnli

lm_eval/tasks/glue/mrpc/default.yaml

-group: glue
+tag: glue
 task: mrpc
 dataset_path: glue
 dataset_name: mrpc

lm_eval/tasks/glue/qnli/default.yaml

-group: glue
+tag: glue
 task: qnli
 dataset_path: glue
 dataset_name: qnli

lm_eval/tasks/glue/qqp/default.yaml

-group: glue
+tag: glue
 task: qqp
 dataset_path: glue
 dataset_name: qqp

lm_eval/tasks/glue/rte/default.yaml

-group: glue
+tag: glue
 task: rte
 dataset_path: glue
 dataset_name: rte

lm_eval/tasks/glue/sst2/default.yaml

-group: glue
+tag: glue
 task: sst2
 dataset_path: glue
 dataset_name: sst2

lm_eval/tasks/glue/wnli/default.yaml

-group: glue
+tag: glue
 task: wnli
 dataset_path: glue
 dataset_name: wnli

lm_eval/tasks/gpqa/README.md

@@ -25,11 +25,15 @@ Homepage: `https://github.com/idavidrein/gpqa/tree/main`
 
 This dataset is gated, so you will have to accept the terms of use at https://huggingface.co/datasets/Idavidrein/gpqa and login via `huggingface-cli login` using your HF Hub token before running this task.
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 
-* `gpqa`
+None
+
+#### Tags
+
+* `gpqa`: runs all GPQA variants.
 
 #### Tasks
 

lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml

 dataset_path: Idavidrein/gpqa
-group: gpqa
+tag: gpqa
 output_type: generate_until
 process_docs: !function utils.process_docs
 training_split: train
@@ -35,4 +35,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml

 dataset_path: Idavidrein/gpqa
-group: gpqa
+tag: gpqa
 output_type: generate_until
 process_docs: !function utils.process_docs
 training_split: train