Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into superglue

lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml

+# Generated by utils.py
+dataset_name: coordinate_structure_constraint_object_extraction
+include: template_yaml
+task: blimp_coordinate_structure_constraint_object_extraction

lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml

+# Generated by utils.py
+dataset_name: determiner_noun_agreement_1
+include: template_yaml
+task: blimp_determiner_noun_agreement_1

lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml

+# Generated by utils.py
+dataset_name: determiner_noun_agreement_2
+include: template_yaml
+task: blimp_determiner_noun_agreement_2

lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml

+# Generated by utils.py
+dataset_name: determiner_noun_agreement_irregular_1
+include: template_yaml
+task: blimp_determiner_noun_agreement_irregular_1

lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml

+# Generated by utils.py
+dataset_name: determiner_noun_agreement_irregular_2
+include: template_yaml
+task: blimp_determiner_noun_agreement_irregular_2

lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml

+# Generated by utils.py
+dataset_name: determiner_noun_agreement_with_adj_2
+include: template_yaml
+task: blimp_determiner_noun_agreement_with_adj_2

lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml

+# Generated by utils.py
+dataset_name: determiner_noun_agreement_with_adj_irregular_1
+include: template_yaml
+task: blimp_determiner_noun_agreement_with_adj_irregular_1

lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml

+# Generated by utils.py
+dataset_name: determiner_noun_agreement_with_adj_irregular_2
+include: template_yaml
+task: blimp_determiner_noun_agreement_with_adj_irregular_2

lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml

+# Generated by utils.py
+dataset_name: determiner_noun_agreement_with_adjective_1
+include: template_yaml
+task: blimp_determiner_noun_agreement_with_adjective_1

lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml

+# Generated by utils.py
+dataset_name: distractor_agreement_relational_noun
+include: template_yaml
+task: blimp_distractor_agreement_relational_noun

lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml

+# Generated by utils.py
+dataset_name: distractor_agreement_relative_clause
+include: template_yaml
+task: blimp_distractor_agreement_relative_clause

lm_eval/tasks/blimp/drop_argument.yaml

+# Generated by utils.py
+dataset_name: drop_argument
+include: template_yaml
+task: blimp_drop_argument

lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml

+# Generated by utils.py
+dataset_name: ellipsis_n_bar_1
+include: template_yaml
+task: blimp_ellipsis_n_bar_1

lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml

+# Generated by utils.py
+dataset_name: ellipsis_n_bar_2
+include: template_yaml
+task: blimp_ellipsis_n_bar_2

lm_eval/tasks/blimp/existential_there_object_raising.yaml

+# Generated by utils.py
+dataset_name: existential_there_object_raising
+include: template_yaml
+task: blimp_existential_there_object_raising

lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml

+# Generated by utils.py
+dataset_name: existential_there_quantifiers_1
+include: template_yaml
+task: blimp_existential_there_quantifiers_1

lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml

+# Generated by utils.py
+dataset_name: existential_there_quantifiers_2
+include: template_yaml
+task: blimp_existential_there_quantifiers_2

lm_eval/tasks/blimp/existential_there_subject_raising.yaml

+# Generated by utils.py
+dataset_name: existential_there_subject_raising
+include: template_yaml
+task: blimp_existential_there_subject_raising

lm_eval/tasks/blimp/expletive_it_object_raising.yaml

+# Generated by utils.py
+dataset_name: expletive_it_object_raising
+include: template_yaml
+task: blimp_expletive_it_object_raising

lm_eval/tasks/blimp/generate_configs.py

+import yaml
+
+all_subtasks = [
+    "adjunct_island",
+    "anaphor_gender_agreement",
+    "anaphor_number_agreement",
+    "animate_subject_passive",
+    "animate_subject_trans",
+    "causative",
+    "complex_NP_island",
+    "coordinate_structure_constraint_complex_left_branch",
+    "coordinate_structure_constraint_object_extraction",
+    "determiner_noun_agreement_1",
+    "determiner_noun_agreement_2",
+    "determiner_noun_agreement_irregular_1",
+    "determiner_noun_agreement_irregular_2",
+    "determiner_noun_agreement_with_adj_2",
+    "determiner_noun_agreement_with_adj_irregular_1",
+    "determiner_noun_agreement_with_adj_irregular_2",
+    "determiner_noun_agreement_with_adjective_1",
+    "distractor_agreement_relational_noun",
+    "distractor_agreement_relative_clause",
+    "drop_argument",
+    "ellipsis_n_bar_1",
+    "ellipsis_n_bar_2",
+    "existential_there_object_raising",
+    "existential_there_quantifiers_1",
+    "existential_there_quantifiers_2",
+    "existential_there_subject_raising",
+    "expletive_it_object_raising",
+    "inchoative",
+    "intransitive",
+    "irregular_past_participle_adjectives",
+    "irregular_past_participle_verbs",
+    "irregular_plural_subject_verb_agreement_1",
+    "irregular_plural_subject_verb_agreement_2",
+    "left_branch_island_echo_question",
+    "left_branch_island_simple_question",
+    "matrix_question_npi_licensor_present",
+    "npi_present_1",
+    "npi_present_2",
+    "only_npi_licensor_present",
+    "only_npi_scope",
+    "passive_1",
+    "passive_2",
+    "principle_A_c_command",
+    "principle_A_case_1",
+    "principle_A_case_2",
+    "principle_A_domain_1",
+    "principle_A_domain_2",
+    "principle_A_domain_3",
+    "principle_A_reconstruction",
+    "regular_plural_subject_verb_agreement_1",
+    "regular_plural_subject_verb_agreement_2",
+    "sentential_negation_npi_licensor_present",
+    "sentential_negation_npi_scope",
+    "sentential_subject_island",
+    "superlative_quantifiers_1",
+    "superlative_quantifiers_2",
+    "tough_vs_raising_1",
+    "tough_vs_raising_2",
+    "transitive",
+    "wh_island",
+    "wh_questions_object_gap",
+    "wh_questions_subject_gap",
+    "wh_questions_subject_gap_long_distance",
+    "wh_vs_that_no_gap",
+    "wh_vs_that_no_gap_long_distance",
+    "wh_vs_that_with_gap",
+    "wh_vs_that_with_gap_long_distance",
+]
+
+
+def main() -> None:
+    for task in all_subtasks:
+
+        file_name = f"{task}.yaml"
+        try:
+            with open(f"{file_name}", "w") as f:
+                f.write("# Generated by utils.py\n")
+                yaml.dump(
+                    {
+                        "include": "template_yaml",
+                        "task": "blimp_" + task,
+                        "dataset_name": task,
+                    },
+                    f,
+                )
+        except FileExistsError:
+            pass
+
+
+if __name__ == "__main__":
+    main()