Merge branch 'big-refactor' of...

Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into num_fewshot_fix

lm_eval/tasks/super_glue/record/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
   - metric: !function t5_utils.f1
     aggregation: !function t5_utils.squad_f1_agg
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/rte/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: ['True', 'False']
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/rte/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/wic/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/wic/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/wsc/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/wsc/t5-prompt.yaml

@@ -19,3 +19,5 @@ filter_list:
   - name: "wsc_postprocessor"
     filter:
       - function: !function t5_utils.WSCPostprocess
+metadata:
+  - version: 0.0

lm_eval/tasks/swag/swag.yaml

@@ -15,3 +15,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/toxigen/toxigen.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/translation/wmt_common_yaml

@@ -13,3 +13,5 @@ generation_kwargs:
   do_sample: false
   temperature: 0.0
 repeats: 1
+metadata:
+  - version: 0.0

lm_eval/tasks/triviaqa/default.yaml

@@ -27,3 +27,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

@@ -75,3 +75,5 @@ metric_list:
   - metric: rougeL_diff
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml

@@ -32,3 +32,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_mc2.yaml

@@ -9,3 +9,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/unscramble/anagrams1.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/anagrams2.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/cycle_letters.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/random_insertion.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/reversed_words.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/webqs/webqs.yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0