Merge branch 'big-refactor' of...

Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into num_fewshot_fix

lm_eval/tasks/mmlu/default/_default_template_yaml

@@ -11,3 +11,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml

@@ -21,3 +21,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml

@@ -21,3 +21,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 0.0

lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml

@@ -12,3 +12,5 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml

@@ -12,3 +12,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml

@@ -10,3 +10,5 @@ should_decontaminate: true
 doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}"
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/model_written_evals/persona/_template_yaml

@@ -8,3 +8,5 @@ doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml

@@ -10,3 +10,5 @@ doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml

@@ -10,3 +10,5 @@ doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

@@ -10,3 +10,5 @@ doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/model_written_evals/winogenerated/_template_yaml

@@ -8,3 +8,5 @@ doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:
   - metric: acc
+metadata:
+  - version: 0.0

lm_eval/tasks/mutual/mutual.yaml

@@ -21,3 +21,5 @@ metric_list:
   - metric: mrr
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/nq_open/nq_open.yaml

@@ -28,3 +28,5 @@ metric_list:
     ignore_punctuation: true
     regexes_to_ignore:
     - "\ban|a|the\b"
+metadata:
+  - version: 0.0

lm_eval/tasks/openbookqa/openbookqa.yaml

@@ -17,3 +17,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/paws-x/pawsx_template_yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/pile/pile_arxiv.yaml

@@ -19,3 +19,5 @@ metric_list:
   - metric: bits_per_byte
     aggregation: bits_per_byte
     higher_is_better: false
+metadata:
+  - version: 2.0

lm_eval/tasks/piqa/piqa.yaml

@@ -17,3 +17,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/polemo2/polemo2_in.yaml

@@ -41,3 +41,5 @@ metric_list:
   - metric: accuracy
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/prost/corypaik_prost.yaml

@@ -15,3 +15,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/pubmedqa/pubmedqa.yaml

@@ -12,3 +12,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0