Merge pull request #1031 from EleutherAI/versioning

[Refactor] Versioning

lm_eval/tasks/qa4mre/qa4mre_2011.yaml

@@ -18,3 +18,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/qasper/bool.yaml

@@ -10,3 +10,5 @@ doc_to_target: 1
 doc_to_choice: ["no", "yes"]
 metric_list:
   - metric: f1
+metadata:
+  - version: 1.0

lm_eval/tasks/qasper/freeform.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: !function metrics.f1_abstractive
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/race/race.yaml

@@ -10,3 +10,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/realtoxicityprompts/realtoxicityprompts.yaml

@@ -13,3 +13,5 @@ generation_kwargs:
     - "\n\n"
   do_sample: false
   temperature: 0.0
+metadata:
+  - version: 0.0

lm_eval/tasks/sciq/sciq.yaml

@@ -17,3 +17,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/scrolls/task.py

@@ -108,7 +108,7 @@ def _num_cpu_cores():
 
 
 class _SCROLLSTask(Task):
-    VERSION = 0
+    VERSION = 1
     DATASET_PATH = "tau/scrolls"
     DATASET_NAME = None
     PRUNE_TOKENIZERS = None

lm_eval/tasks/siqa/default.yml

@@ -12,3 +12,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/squadv2/task.py

@@ -50,7 +50,7 @@ def _squad_agg(key, items):
 
 @register_task("squadv2")
 class SQuAD2(Task):
-    VERSION = 1
+    VERSION = 2
     DATASET_PATH = "squad_v2"
     DATASET_NAME = None
 

lm_eval/tasks/storycloze/storycloze_2016.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/boolq/default.yaml

@@ -13,3 +13,5 @@ should_decontaminate: true
 doc_to_decontamination_query: passage
 metric_list:
   - metric: acc
+metadata:
+  - version: 2.0

lm_eval/tasks/super_glue/boolq/seq2seq.yaml

@@ -22,3 +22,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/boolq/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/cb/default.yaml

@@ -13,3 +13,5 @@ metric_list:
   - metric: acc
   - metric: f1
     aggregation: !function "aggregate.cb_multi_fi"
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/cb/t5-prompt.yaml

@@ -21,3 +21,5 @@ metric_list:
   - metric: !function "t5_utils.mean_3class_f1"
     aggregation: !function "t5_utils.agg_mean_3class_f1"
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/copa/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: !function utils.doc_to_target
 doc_to_choice: !function utils.doc_to_choice
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/copa/t5-prompt.yaml

@@ -18,3 +18,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/multirc/default.yaml

@@ -11,3 +11,5 @@ doc_to_target: label
 doc_to_choice: "['''{{answer}}\\nIs the answer correct? yes''', '''{{answer}}\\nIs the answer correct? no''']"
 metric_list:
   - metric: acc
+metadata:
+  - version: 2.0

lm_eval/tasks/super_glue/multirc/t5-prompt.yaml

@@ -19,3 +19,5 @@ metric_list:
   - metric: !function t5_utils.em
     aggregation: !function t5_utils.agg_em
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/super_glue/record/default.yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: em
     higher_is_better: True
     aggregation: mean
+metadata:
+  - version: 1.0