Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into superglue

lm_eval/tasks/blimp/inchoative.yaml

+# Generated by utils.py
+dataset_name: inchoative
+include: template_yaml
+task: blimp_inchoative

lm_eval/tasks/blimp/intransitive.yaml

+# Generated by utils.py
+dataset_name: intransitive
+include: template_yaml
+task: blimp_intransitive

lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml

+# Generated by utils.py
+dataset_name: irregular_past_participle_adjectives
+include: template_yaml
+task: blimp_irregular_past_participle_adjectives

lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml

+# Generated by utils.py
+dataset_name: irregular_past_participle_verbs
+include: template_yaml
+task: blimp_irregular_past_participle_verbs

lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml

+# Generated by utils.py
+dataset_name: irregular_plural_subject_verb_agreement_1
+include: template_yaml
+task: blimp_irregular_plural_subject_verb_agreement_1

lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml

+# Generated by utils.py
+dataset_name: irregular_plural_subject_verb_agreement_2
+include: template_yaml
+task: blimp_irregular_plural_subject_verb_agreement_2

lm_eval/tasks/blimp/left_branch_island_echo_question.yaml

+# Generated by utils.py
+dataset_name: left_branch_island_echo_question
+include: template_yaml
+task: blimp_left_branch_island_echo_question

lm_eval/tasks/blimp/left_branch_island_simple_question.yaml

+# Generated by utils.py
+dataset_name: left_branch_island_simple_question
+include: template_yaml
+task: blimp_left_branch_island_simple_question

lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml

+# Generated by utils.py
+dataset_name: matrix_question_npi_licensor_present
+include: template_yaml
+task: blimp_matrix_question_npi_licensor_present

lm_eval/tasks/blimp/npi_present_1.yaml

+# Generated by utils.py
+dataset_name: npi_present_1
+include: template_yaml
+task: blimp_npi_present_1

lm_eval/tasks/blimp/npi_present_2.yaml

+# Generated by utils.py
+dataset_name: npi_present_2
+include: template_yaml
+task: blimp_npi_present_2

lm_eval/tasks/blimp/only_npi_licensor_present.yaml

+# Generated by utils.py
+dataset_name: only_npi_licensor_present
+include: template_yaml
+task: blimp_only_npi_licensor_present

lm_eval/tasks/blimp/only_npi_scope.yaml

+# Generated by utils.py
+dataset_name: only_npi_scope
+include: template_yaml
+task: blimp_only_npi_scope

lm_eval/tasks/blimp/passive_1.yaml

+# Generated by utils.py
+dataset_name: passive_1
+include: template_yaml
+task: blimp_passive_1

lm_eval/tasks/blimp/passive_2.yaml

+# Generated by utils.py
+dataset_name: passive_2
+include: template_yaml
+task: blimp_passive_2

lm_eval/tasks/blimp/principle_A_c_command.yaml

+# Generated by utils.py
+dataset_name: principle_A_c_command
+include: template_yaml
+task: blimp_principle_A_c_command

lm_eval/tasks/blimp/principle_A_case_1.yaml

+# Generated by utils.py
+dataset_name: principle_A_case_1
+include: template_yaml
+task: blimp_principle_A_case_1

lm_eval/tasks/blimp/principle_A_case_2.yaml

+# Generated by utils.py
+dataset_name: principle_A_case_2
+include: template_yaml
+task: blimp_principle_A_case_2

lm_eval/tasks/blimp/principle_A_domain_1.yaml

+# Generated by utils.py
+dataset_name: principle_A_domain_1
+include: template_yaml
+task: blimp_principle_A_domain_1

lm_eval/tasks/blimp/principle_A_domain_2.yaml

+# Generated by utils.py
+dataset_name: principle_A_domain_2
+include: template_yaml
+task: blimp_principle_A_domain_2