Merge branch 'main' into multimodal-prototyping

lm_eval/tasks/unscramble/cycle_letters.yaml

-group:
+tag:
   - unscramble
 task: cycle_letters
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/unscramble/random_insertion.yaml

-group:
+tag:
   - unscramble
 task: random_insertion
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/unscramble/reversed_words.yaml

-group:
+tag:
   - unscramble
 task: reversed_words
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/webqs/webqs.yaml

-group:
+tag:
   - freebase
 task: webqs
 dataset_path: web_questions

lm_eval/tasks/wmdp/README.md

@@ -24,7 +24,7 @@ Homepage: https://wmdp.ai
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 

lm_eval/tasks/wmdp/_default_template_yaml

 dataset_path: cais/wmdp
-group: wmdp
 test_split: test
 training_split: null
 validation_split: null

lm_eval/tasks/wmdp/_wmdp.yaml

+group: wmdp
+task:
+  - wmdp_bio
+  - wmdp_chem
+  - wmdp_cyber
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/wmt2016/README.md

@@ -27,11 +27,7 @@ Homepage: https://huggingface.co/datasets/wmt16
 }
 ```
 
-### Groups and Tasks
-
-#### Groups
-
-* `wmt-t5-prompt`: Group for all wmt tasks with prompt templates used for T5 (`Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`)
+### Groups, Tags, and Tasks
 
 #### Tasks
 

lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml

-group:
-  - wmt-t5-prompt
 task: wmt-ro-en-t5-prompt
 dataset_path: wmt16
 dataset_name: ro-en

lm_eval/tasks/xcopa/_xcopa.yaml

+group: xcopa
+task:
+  - xcopa_et
+  - xcopa_ht
+  - xcopa_id
+  - xcopa_it
+  - xcopa_qu
+  - xcopa_sw
+  - xcopa_ta
+  - xcopa_th
+  - xcopa_tr
+  - xcopa_vi
+  - xcopa_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: True
+metadata:
+  version: 1.0

lm_eval/tasks/xcopa/default_et.yaml

-group: xcopa
 task: xcopa_et
 dataset_path: xcopa
 dataset_name: et

lm_eval/tasks/xnli/_xnli.yaml

+group: xnli
+task:
+  - xnli_ar
+  - xnli_bg
+  - xnli_de
+  - xnli_el
+  - xnli_en
+  - xnli_es
+  - xnli_fr
+  - xnli_hi
+  - xnli_ru
+  - xnli_sw
+  - xnli_th
+  - xnli_tr
+  - xnli_ur
+  - xnli_vi
+  - xnli_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0

lm_eval/tasks/xnli/xnli_common_yaml

 # This file will be included in the generated language-specific task configs.
 # It doesn't have a yaml file extension as it is not meant to be imported directly
 # by the harness.
-group: xnli
 task: null
 dataset_path: xnli
 dataset_name: null

lm_eval/tasks/xnli_eu/README.md

@@ -24,9 +24,9 @@ Homepage: https://github.com/hitz-zentroa/xnli-eu
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
-#### Groups
+#### Tags
 
 * `xnli_eu_mt_native`: Includes MT and Native variants of the XNLIeu dataset.
 

lm_eval/tasks/xnli_eu/xnli_common_yaml

-group: xnli
 task: null
 dataset_path: xnli
 dataset_name: null

lm_eval/tasks/xnli_eu/xnli_eu_mt.yaml

 include: xnli_eu.yaml
-group: xnli_eu_mt_native
+tag: xnli_eu_mt_native
 task: xnli_eu_mt
 dataset_name: eu_mt

lm_eval/tasks/xnli_eu/xnli_eu_native.yaml

 include: xnli_eu.yaml
-group: xnli_eu_mt_native
+tag: xnli_eu_mt_native
 task: xnli_eu_native
 training_split: null
 validation_split: null

lm_eval/tasks/xstorycloze/_xstorycloze.yaml

+group: xstorycloze
+task:
+  - xstorycloze_ar
+  - xstorycloze_en
+  - xstorycloze_es
+  - xstorycloze_eu
+  - xstorycloze_hi
+  - xstorycloze_id
+  - xstorycloze_my
+  - xstorycloze_ru
+  - xstorycloze_sw
+  - xstorycloze_te
+  - xstorycloze_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0

lm_eval/tasks/xstorycloze/default_ar.yaml

-group: xstorycloze
 task: xstorycloze_ar
 dataset_path: juletxara/xstory_cloze
 dataset_name: ar

lm_eval/tasks/xwinograd/_xwinograd.yaml

+group: xwinograd
+task:
+  - xwinograd_en
+  - xwinograd_fr
+  - xwinograd_jp
+  - xwinograd_pt
+  - xwinograd_ru
+  - xwinograd_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0