merged latest and added altworld files

lm_eval/tasks/bigbench/multiple_choice_template_yaml

@@ -11,3 +11,5 @@ doc_to_choice: "{{multiple_choice_targets}}"
 metric_list:
   - metric: acc
   # TODO: brier score and other metrics
+metadata:
+  - version: 0.0

lm_eval/tasks/blimp/template_yaml → lm_eval/tasks/blimp/_template_yaml

@@ -5,7 +5,10 @@ validation_split: train
 doc_to_text: ""
 doc_to_target: 0
 doc_to_choice: "{{[sentence_good, sentence_bad]}}"
+num_fewshot: 0
 should_decontaminate: true
 doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}"
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/blimp/adjunct_island.yaml

 # Generated by utils.py
 dataset_name: adjunct_island
-include: template_yaml
+include: _template_yaml
 task: blimp_adjunct_island

lm_eval/tasks/blimp/anaphor_gender_agreement.yaml

 # Generated by utils.py
 dataset_name: anaphor_gender_agreement
-include: template_yaml
+include: _template_yaml
 task: blimp_anaphor_gender_agreement

lm_eval/tasks/blimp/anaphor_number_agreement.yaml

 # Generated by utils.py
 dataset_name: anaphor_number_agreement
-include: template_yaml
+include: _template_yaml
 task: blimp_anaphor_number_agreement

lm_eval/tasks/blimp/animate_subject_passive.yaml

 # Generated by utils.py
 dataset_name: animate_subject_passive
-include: template_yaml
+include: _template_yaml
 task: blimp_animate_subject_passive

lm_eval/tasks/blimp/animate_subject_trans.yaml

 # Generated by utils.py
 dataset_name: animate_subject_trans
-include: template_yaml
+include: _template_yaml
 task: blimp_animate_subject_trans

lm_eval/tasks/blimp/causative.yaml

 # Generated by utils.py
 dataset_name: causative
-include: template_yaml
+include: _template_yaml
 task: blimp_causative

lm_eval/tasks/blimp/complex_NP_island.yaml

 # Generated by utils.py
 dataset_name: complex_NP_island
-include: template_yaml
+include: _template_yaml
 task: blimp_complex_NP_island

lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml

 # Generated by utils.py
 dataset_name: coordinate_structure_constraint_complex_left_branch
-include: template_yaml
+include: _template_yaml
 task: blimp_coordinate_structure_constraint_complex_left_branch

lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml

 # Generated by utils.py
 dataset_name: coordinate_structure_constraint_object_extraction
-include: template_yaml
+include: _template_yaml
 task: blimp_coordinate_structure_constraint_object_extraction

lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml

 # Generated by utils.py
 dataset_name: determiner_noun_agreement_1
-include: template_yaml
+include: _template_yaml
 task: blimp_determiner_noun_agreement_1

lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml

 # Generated by utils.py
 dataset_name: determiner_noun_agreement_2
-include: template_yaml
+include: _template_yaml
 task: blimp_determiner_noun_agreement_2

lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml

 # Generated by utils.py
 dataset_name: determiner_noun_agreement_irregular_1
-include: template_yaml
+include: _template_yaml
 task: blimp_determiner_noun_agreement_irregular_1

lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml

 # Generated by utils.py
 dataset_name: determiner_noun_agreement_irregular_2
-include: template_yaml
+include: _template_yaml
 task: blimp_determiner_noun_agreement_irregular_2

lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml

 # Generated by utils.py
 dataset_name: determiner_noun_agreement_with_adj_2
-include: template_yaml
+include: _template_yaml
 task: blimp_determiner_noun_agreement_with_adj_2

lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml

 # Generated by utils.py
 dataset_name: determiner_noun_agreement_with_adj_irregular_1
-include: template_yaml
+include: _template_yaml
 task: blimp_determiner_noun_agreement_with_adj_irregular_1

lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml

 # Generated by utils.py
 dataset_name: determiner_noun_agreement_with_adj_irregular_2
-include: template_yaml
+include: _template_yaml
 task: blimp_determiner_noun_agreement_with_adj_irregular_2

lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml

 # Generated by utils.py
 dataset_name: determiner_noun_agreement_with_adjective_1
-include: template_yaml
+include: _template_yaml
 task: blimp_determiner_noun_agreement_with_adjective_1

lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml

 # Generated by utils.py
 dataset_name: distractor_agreement_relational_noun
-include: template_yaml
+include: _template_yaml
 task: blimp_distractor_agreement_relational_noun