Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into superglue

lm_eval/tasks/blimp/principle_A_domain_3.yaml

+# Generated by utils.py
+dataset_name: principle_A_domain_3
+include: template_yaml
+task: blimp_principle_A_domain_3

lm_eval/tasks/blimp/principle_A_reconstruction.yaml

+# Generated by utils.py
+dataset_name: principle_A_reconstruction
+include: template_yaml
+task: blimp_principle_A_reconstruction

lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml

+# Generated by utils.py
+dataset_name: regular_plural_subject_verb_agreement_1
+include: template_yaml
+task: blimp_regular_plural_subject_verb_agreement_1

lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml

+# Generated by utils.py
+dataset_name: regular_plural_subject_verb_agreement_2
+include: template_yaml
+task: blimp_regular_plural_subject_verb_agreement_2

lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml

+# Generated by utils.py
+dataset_name: sentential_negation_npi_licensor_present
+include: template_yaml
+task: blimp_sentential_negation_npi_licensor_present

lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml

+# Generated by utils.py
+dataset_name: sentential_negation_npi_scope
+include: template_yaml
+task: blimp_sentential_negation_npi_scope

lm_eval/tasks/blimp/sentential_subject_island.yaml

+# Generated by utils.py
+dataset_name: sentential_subject_island
+include: template_yaml
+task: blimp_sentential_subject_island

lm_eval/tasks/blimp/superlative_quantifiers_1.yaml

+# Generated by utils.py
+dataset_name: superlative_quantifiers_1
+include: template_yaml
+task: blimp_superlative_quantifiers_1

lm_eval/tasks/blimp/superlative_quantifiers_2.yaml

+# Generated by utils.py
+dataset_name: superlative_quantifiers_2
+include: template_yaml
+task: blimp_superlative_quantifiers_2

lm_eval/tasks/blimp/template_yaml

+group: blimp
+dataset_path: blimp
+output_type: multiple_choice
+validation_split: validation
+doc_to_text: ""
+doc_to_target: 0
+doc_to_choice: "{{[sentence_good, sentence_bad]}}"
+should_decontaminate: true
+doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}"
+metric_list:
+  - metric: acc

lm_eval/tasks/blimp/tough_vs_raising_1.yaml

+# Generated by utils.py
+dataset_name: tough_vs_raising_1
+include: template_yaml
+task: blimp_tough_vs_raising_1

lm_eval/tasks/blimp/tough_vs_raising_2.yaml

+# Generated by utils.py
+dataset_name: tough_vs_raising_2
+include: template_yaml
+task: blimp_tough_vs_raising_2

lm_eval/tasks/blimp/transitive.yaml

+# Generated by utils.py
+dataset_name: transitive
+include: template_yaml
+task: blimp_transitive

lm_eval/tasks/blimp/wh_island.yaml

+# Generated by utils.py
+dataset_name: wh_island
+include: template_yaml
+task: blimp_wh_island

lm_eval/tasks/blimp/wh_questions_object_gap.yaml

+# Generated by utils.py
+dataset_name: wh_questions_object_gap
+include: template_yaml
+task: blimp_wh_questions_object_gap

lm_eval/tasks/blimp/wh_questions_subject_gap.yaml

+# Generated by utils.py
+dataset_name: wh_questions_subject_gap
+include: template_yaml
+task: blimp_wh_questions_subject_gap

lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml

+# Generated by utils.py
+dataset_name: wh_questions_subject_gap_long_distance
+include: template_yaml
+task: blimp_wh_questions_subject_gap_long_distance

lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml

+# Generated by utils.py
+dataset_name: wh_vs_that_no_gap
+include: template_yaml
+task: blimp_wh_vs_that_no_gap

lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml

+# Generated by utils.py
+dataset_name: wh_vs_that_no_gap_long_distance
+include: template_yaml
+task: blimp_wh_vs_that_no_gap_long_distance

lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml

+# Generated by utils.py
+dataset_name: wh_vs_that_with_gap
+include: template_yaml
+task: blimp_wh_vs_that_with_gap