merged latest and added altworld files

lm_eval/tasks/super_glue/wic/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/wsc/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/wsc/t5-prompt.yaml

@@ -19,3 +19,5 @@ filter_list:
   - name: "wsc_postprocessor"
     filter:
       - function: !function t5_utils.WSCPostprocess
+metadata:
+  - version: 0.0

lm_eval/tasks/swag/swag.yaml

@@ -15,3 +15,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/toxigen/toxigen.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/translation/wmt_common_yaml

@@ -13,3 +13,5 @@ generation_kwargs:
   do_sample: false
   temperature: 0.0
 repeats: 1
+metadata:
+  - version: 0.0

lm_eval/tasks/triviaqa/default.yaml

@@ -27,3 +27,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/alternative_worlds/truthfulqa_mc1.yaml

+group:
+  - truthfulqa
+task: truthfulqa_mc1
+dataset_path: truthful_qa
+dataset_name: multiple_choice
+output_type: multiple_choice
+training_split: null
+validation_split: validation
+test_split: null
+num_fewshot: 0
+doc_to_text: "{{question}}"
+doc_to_target: 0
+doc_to_choice: "{{mc1_targets.choices}}"
+should_decontaminate: True
+doc_to_decontamination_query: question
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/truthfulqa/alternative_worlds/truthfulqa_mc2.yaml

+include: truthfulqa_mc1.yaml
+task: truthfulqa_mc2
+doc_to_target: 0
+doc_to_choice: "{{mc2_targets.choices}}"
+process_results: !function utils.process_results_mc2
+should_decontaminate: True
+doc_to_decontamination_query: question
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

@@ -75,3 +75,5 @@ metric_list:
   - metric: rougeL_diff
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml

@@ -32,3 +32,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_mc2.yaml

@@ -9,3 +9,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/unscramble/anagrams1.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/anagrams2.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/cycle_letters.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/random_insertion.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/reversed_words.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/webqs/webqs.yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/wikitext/wikitext.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: word_perplexity
   - metric: byte_perplexity
   - metric: bits_per_byte
+metadata:
+  - version: 2.0

lm_eval/tasks/winogrande/default.yaml

@@ -13,3 +13,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0