Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into multiprompt

lm_eval/tasks/translation/wmt14_en-fr.yaml

@@ -5,8 +5,7 @@ doc_to_target: ' {{translation["fr"]}}'
 doc_to_text: 'English phrase: {{translation["en"]}}
 
   French phrase:'
-group:
-- generate_until
+tag:
 - translation
 - wmt14
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt14_fr-en.yaml

@@ -5,8 +5,7 @@ doc_to_target: ' {{translation["en"]}}'
 doc_to_text: 'French phrase: {{translation["fr"]}}
 
   English phrase:'
-group:
-- generate_until
+tag:
 - translation
 - wmt14
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt16_de-en.yaml

@@ -5,8 +5,7 @@ doc_to_target: ' {{translation["en"]}}'
 doc_to_text: 'German phrase: {{translation["de"]}}
 
   English phrase:'
-group:
-- generate_until
+tag:
 - translation
 - wmt16
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt16_en-de.yaml

@@ -5,8 +5,7 @@ doc_to_target: ' {{translation["de"]}}'
 doc_to_text: 'English phrase: {{translation["en"]}}
 
   German phrase:'
-group:
-- generate_until
+tag:
 - translation
 - wmt16
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt16_en-ro.yaml

@@ -5,8 +5,7 @@ doc_to_target: ' {{translation["ro"]}}'
 doc_to_text: 'English phrase: {{translation["en"]}}
 
   Romanian phrase:'
-group:
-- generate_until
+tag:
 - translation
 - wmt16
 - gpt3_translation_benchmarks

lm_eval/tasks/translation/wmt16_ro-en.yaml

@@ -5,8 +5,7 @@ doc_to_target: ' {{translation["en"]}}'
 doc_to_text: 'Romanian phrase: {{translation["ro"]}}
 
   English phrase:'
-group:
-- generate_until
+tag:
 - translation
 - wmt16
 - gpt3_translation_benchmarks

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

-group:
+tag:
   - truthfulqa
 task: truthfulqa_gen
 dataset_path: truthful_qa

lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml

-group:
+tag:
   - truthfulqa
 task: truthfulqa_mc1
 dataset_path: truthful_qa

lm_eval/tasks/unscramble/anagrams1.yaml

-group:
+tag:
   - unscramble
 task: anagrams1
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/unscramble/anagrams2.yaml

-group:
+tag:
   - unscramble
 task: anagrams2
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/unscramble/cycle_letters.yaml

-group:
+tag:
   - unscramble
 task: cycle_letters
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/unscramble/random_insertion.yaml

-group:
+tag:
   - unscramble
 task: random_insertion
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/unscramble/reversed_words.yaml

-group:
+tag:
   - unscramble
 task: reversed_words
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/webqs/webqs.yaml

-group:
+tag:
   - freebase
 task: webqs
 dataset_path: web_questions

lm_eval/tasks/wmdp/README.md

@@ -24,7 +24,7 @@ Homepage: https://wmdp.ai
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 

lm_eval/tasks/wmdp/_default_template_yaml

 dataset_path: cais/wmdp
-group: wmdp
 test_split: test
 training_split: null
 validation_split: null

lm_eval/tasks/wmdp/_wmdp.yaml

+group: wmdp
+task:
+  - wmdp_bio
+  - wmdp_chem
+  - wmdp_cyber
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/wmt2016/README.md

@@ -27,11 +27,7 @@ Homepage: https://huggingface.co/datasets/wmt16
 }
 ```
 
-### Groups and Tasks
-
-#### Groups
-
-* `wmt-t5-prompt`: Group for all wmt tasks with prompt templates used for T5 (`Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`)
+### Groups, Tags, and Tasks
 
 #### Tasks
 

lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml

-group:
-  - wmt-t5-prompt
 task: wmt-ro-en-t5-prompt
 dataset_path: wmt16
 dataset_name: ro-en

lm_eval/tasks/xcopa/_xcopa.yaml

+group: xcopa
+task:
+  - xcopa_et
+  - xcopa_ht
+  - xcopa_id
+  - xcopa_it
+  - xcopa_qu
+  - xcopa_sw
+  - xcopa_ta
+  - xcopa_th
+  - xcopa_tr
+  - xcopa_vi
+  - xcopa_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: True
+metadata:
+  version: 1.0