Merge remote-tracking branch 'origin/big-refactor' into big-refactor_dp

lm_eval/tasks/super_glue/boolq/default.yaml

@@ -13,3 +13,5 @@ should_decontaminate: true
 doc_to_decontamination_query: passage
 metric_list:
   - metric: acc
+metadata:
+  - version: 2.0

lm_eval/tasks/super_glue/boolq/seq2seq.yaml

@@ -22,3 +22,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/boolq/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/cb/default.yaml

@@ -13,3 +13,5 @@ metric_list:
   - metric: acc
   - metric: f1
     aggregation: !function "aggregate.cb_multi_fi"
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/cb/t5-prompt.yaml

@@ -21,3 +21,5 @@ metric_list:
   - metric: !function "t5_utils.mean_3class_f1"
     aggregation: !function "t5_utils.agg_mean_3class_f1"
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/copa/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: !function utils.doc_to_target
 doc_to_choice: !function utils.doc_to_choice
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/copa/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/multirc/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: "['''{{answer}}\\nIs the answer correct? yes''', '''{{answer}}\\nIs the answer correct? no''']"
 metric_list:
   - metric: acc
+metadata:
+  - version: 2.0

lm_eval/tasks/super_glue/multirc/t5-prompt.yaml

@@ -19,3 +19,5 @@ metric_list:
   - metric: !function t5_utils.em
     aggregation: !function t5_utils.agg_em
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/record/default.yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: em
     higher_is_better: True
     aggregation: mean
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/record/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
   - metric: !function t5_utils.f1
     aggregation: !function t5_utils.squad_f1_agg
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/rte/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: ['True', 'False']
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/rte/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/wic/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/wic/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/wsc/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/wsc/t5-prompt.yaml

@@ -19,3 +19,5 @@ filter_list:
   - name: "wsc_postprocessor"
     filter:
       - function: !function t5_utils.WSCPostprocess
+metadata:
+  - version: 0.0

lm_eval/tasks/swag/swag.yaml

@@ -15,3 +15,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/toxigen/toxigen.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/translation/wmt_common_yaml

@@ -13,3 +13,5 @@ generation_kwargs:
   do_sample: false
   temperature: 0.0
 repeats: 1
+metadata:
+  - version: 0.0