Merge remote-tracking branch 'origin/big-refactor' into big-refactor_dp

lm_eval/tasks/model_written_evals/winogenerated/_template_yaml

@@ -8,3 +8,5 @@ doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/mutual/mutual.yaml

@@ -21,3 +21,5 @@ metric_list:
   - metric: mrr
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/nq_open/nq_open.yaml

@@ -28,3 +28,5 @@ metric_list:
     ignore_punctuation: true
     regexes_to_ignore:
     - "\ban|a|the\b"
+metadata:
+  - version: 0.0

lm_eval/tasks/openbookqa/openbookqa.yaml

@@ -17,3 +17,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/paws-x/pawsx_template_yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/pile/pile_arxiv.yaml

@@ -19,3 +19,5 @@ metric_list:
   - metric: bits_per_byte
     aggregation: bits_per_byte
     higher_is_better: false
+metadata:
+  - version: 2.0

lm_eval/tasks/piqa/piqa.yaml

@@ -17,3 +17,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/polemo2/polemo2_in.yaml

@@ -41,3 +41,5 @@ metric_list:
   - metric: accuracy
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/prost/corypaik_prost.yaml

@@ -15,3 +15,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/pubmedqa/pubmedqa.yaml

@@ -12,3 +12,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/qa4mre/qa4mre_2011.yaml

@@ -18,3 +18,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/qasper/bool.yaml

@@ -10,3 +10,5 @@ doc_to_target: 1
 doc_to_choice: ["no", "yes"]
 metric_list:
   - metric: f1
+metadata:
+  - version: 1.0

lm_eval/tasks/qasper/freeform.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: !function metrics.f1_abstractive
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/race/race.yaml

@@ -10,3 +10,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/realtoxicityprompts/realtoxicityprompts.yaml

@@ -13,3 +13,5 @@ generation_kwargs:
     - "\n\n"
   do_sample: false
   temperature: 0.0
+metadata:
+  - version: 0.0

lm_eval/tasks/sciq/sciq.yaml

@@ -17,3 +17,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/scrolls/task.py

@@ -108,7 +108,7 @@ def _num_cpu_cores():
 
 
 class _SCROLLSTask(Task):
-    VERSION = 0
+    VERSION = 1
     DATASET_PATH = "tau/scrolls"
     DATASET_NAME = None
     PRUNE_TOKENIZERS = None

lm_eval/tasks/siqa/bigbench.yml → lm_eval/tasks/siqa/default.yml

@@ -12,3 +12,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/squadv2/task.py

@@ -50,7 +50,7 @@ def _squad_agg(key, items):
 
 @register_task("squadv2")
 class SQuAD2(Task):
-    VERSION = 1
+    VERSION = 2
     DATASET_PATH = "squad_v2"
     DATASET_NAME = None
 

lm_eval/tasks/storycloze/storycloze_2016.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0