add blimp

lm_eval/tasks/blimp/drop_argument.yaml

+# Generated by utils.py
+dataset_name: drop_argument
+include: template_yaml
+task: blimp_drop_argument

lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml

+# Generated by utils.py
+dataset_name: ellipsis_n_bar_1
+include: template_yaml
+task: blimp_ellipsis_n_bar_1

lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml

+# Generated by utils.py
+dataset_name: ellipsis_n_bar_2
+include: template_yaml
+task: blimp_ellipsis_n_bar_2

lm_eval/tasks/blimp/existential_there_object_raising.yaml

+# Generated by utils.py
+dataset_name: existential_there_object_raising
+include: template_yaml
+task: blimp_existential_there_object_raising

lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml

+# Generated by utils.py
+dataset_name: existential_there_quantifiers_1
+include: template_yaml
+task: blimp_existential_there_quantifiers_1

lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml

+# Generated by utils.py
+dataset_name: existential_there_quantifiers_2
+include: template_yaml
+task: blimp_existential_there_quantifiers_2

lm_eval/tasks/blimp/existential_there_subject_raising.yaml

+# Generated by utils.py
+dataset_name: existential_there_subject_raising
+include: template_yaml
+task: blimp_existential_there_subject_raising

lm_eval/tasks/blimp/expletive_it_object_raising.yaml

+# Generated by utils.py
+dataset_name: expletive_it_object_raising
+include: template_yaml
+task: blimp_expletive_it_object_raising

lm_eval/tasks/blimp/generate_configs.py

+
+import yaml
+
+all_subtasks = [
+    'adjunct_island',
+    'anaphor_gender_agreement',
+    'anaphor_number_agreement',
+    'animate_subject_passive',
+    'animate_subject_trans',
+    'causative',
+    'complex_NP_island',
+    'coordinate_structure_constraint_complex_left_branch',
+    'coordinate_structure_constraint_object_extraction',
+    'determiner_noun_agreement_1',
+    'determiner_noun_agreement_2',
+    'determiner_noun_agreement_irregular_1',
+    'determiner_noun_agreement_irregular_2',
+    'determiner_noun_agreement_with_adj_2',
+    'determiner_noun_agreement_with_adj_irregular_1',
+    'determiner_noun_agreement_with_adj_irregular_2',
+    'determiner_noun_agreement_with_adjective_1',
+    'distractor_agreement_relational_noun',
+    'distractor_agreement_relative_clause',
+    'drop_argument',
+    'ellipsis_n_bar_1',
+    'ellipsis_n_bar_2',
+    'existential_there_object_raising',
+    'existential_there_quantifiers_1',
+    'existential_there_quantifiers_2',
+    'existential_there_subject_raising',
+    'expletive_it_object_raising',
+    'inchoative',
+    'intransitive',
+    'irregular_past_participle_adjectives',
+    'irregular_past_participle_verbs',
+    'irregular_plural_subject_verb_agreement_1',
+    'irregular_plural_subject_verb_agreement_2',
+    'left_branch_island_echo_question',
+    'left_branch_island_simple_question',
+    'matrix_question_npi_licensor_present',
+    'npi_present_1',
+    'npi_present_2',
+    'only_npi_licensor_present',
+    'only_npi_scope',
+    'passive_1',
+    'passive_2',
+    'principle_A_c_command',
+    'principle_A_case_1',
+    'principle_A_case_2',
+    'principle_A_domain_1',
+    'principle_A_domain_2',
+    'principle_A_domain_3',
+    'principle_A_reconstruction',
+    'regular_plural_subject_verb_agreement_1',
+    'regular_plural_subject_verb_agreement_2',
+    'sentential_negation_npi_licensor_present',
+    'sentential_negation_npi_scope',
+    'sentential_subject_island',
+    'superlative_quantifiers_1',
+    'superlative_quantifiers_2',
+    'tough_vs_raising_1',
+    'tough_vs_raising_2',
+    'transitive',
+    'wh_island',
+    'wh_questions_object_gap',
+    'wh_questions_subject_gap',
+    'wh_questions_subject_gap_long_distance',
+    'wh_vs_that_no_gap',
+    'wh_vs_that_no_gap_long_distance',
+    'wh_vs_that_with_gap',
+    'wh_vs_that_with_gap_long_distance'
+]
+
+
+def main() -> None:
+    for task in all_subtasks:
+
+        file_name = f"{task}.yaml"
+        try:
+            with open(f"{file_name}", "w") as f:
+                f.write("# Generated by utils.py\n")
+                yaml.dump(
+                    {
+                        "include": "template_yaml",
+                        "task": "blimp_"+task,
+                        "dataset_name": task,
+                    },
+                    f,
+                )
+        except FileExistsError:
+            pass
+
+
+if __name__ == "__main__":
+    main()

lm_eval/tasks/blimp/inchoative.yaml

+# Generated by utils.py
+dataset_name: inchoative
+include: template_yaml
+task: blimp_inchoative

lm_eval/tasks/blimp/intransitive.yaml

+# Generated by utils.py
+dataset_name: intransitive
+include: template_yaml
+task: blimp_intransitive

lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml

+# Generated by utils.py
+dataset_name: irregular_past_participle_adjectives
+include: template_yaml
+task: blimp_irregular_past_participle_adjectives

lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml

+# Generated by utils.py
+dataset_name: irregular_past_participle_verbs
+include: template_yaml
+task: blimp_irregular_past_participle_verbs

lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml

+# Generated by utils.py
+dataset_name: irregular_plural_subject_verb_agreement_1
+include: template_yaml
+task: blimp_irregular_plural_subject_verb_agreement_1

lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml

+# Generated by utils.py
+dataset_name: irregular_plural_subject_verb_agreement_2
+include: template_yaml
+task: blimp_irregular_plural_subject_verb_agreement_2

lm_eval/tasks/blimp/left_branch_island_echo_question.yaml

+# Generated by utils.py
+dataset_name: left_branch_island_echo_question
+include: template_yaml
+task: blimp_left_branch_island_echo_question

lm_eval/tasks/blimp/left_branch_island_simple_question.yaml

+# Generated by utils.py
+dataset_name: left_branch_island_simple_question
+include: template_yaml
+task: blimp_left_branch_island_simple_question

lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml

+# Generated by utils.py
+dataset_name: matrix_question_npi_licensor_present
+include: template_yaml
+task: blimp_matrix_question_npi_licensor_present

lm_eval/tasks/blimp/npi_present_1.yaml

+# Generated by utils.py
+dataset_name: npi_present_1
+include: template_yaml
+task: blimp_npi_present_1

lm_eval/tasks/blimp/npi_present_2.yaml

+# Generated by utils.py
+dataset_name: npi_present_2
+include: template_yaml
+task: blimp_npi_present_2