Merge pull request #985 from EleutherAI/fewshot-in-table

[Refactor] Num_fewshot process

lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml

 # Generated by utils.py
 dataset_name: distractor_agreement_relational_noun
-include: template_yaml
+include: _template_yaml
 task: blimp_distractor_agreement_relational_noun

lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml

 # Generated by utils.py
 dataset_name: distractor_agreement_relative_clause
-include: template_yaml
+include: _template_yaml
 task: blimp_distractor_agreement_relative_clause

lm_eval/tasks/blimp/drop_argument.yaml

 # Generated by utils.py
 dataset_name: drop_argument
-include: template_yaml
+include: _template_yaml
 task: blimp_drop_argument

lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml

 # Generated by utils.py
 dataset_name: ellipsis_n_bar_1
-include: template_yaml
+include: _template_yaml
 task: blimp_ellipsis_n_bar_1

lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml

 # Generated by utils.py
 dataset_name: ellipsis_n_bar_2
-include: template_yaml
+include: _template_yaml
 task: blimp_ellipsis_n_bar_2

lm_eval/tasks/blimp/existential_there_object_raising.yaml

 # Generated by utils.py
 dataset_name: existential_there_object_raising
-include: template_yaml
+include: _template_yaml
 task: blimp_existential_there_object_raising

lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml

 # Generated by utils.py
 dataset_name: existential_there_quantifiers_1
-include: template_yaml
+include: _template_yaml
 task: blimp_existential_there_quantifiers_1

lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml

 # Generated by utils.py
 dataset_name: existential_there_quantifiers_2
-include: template_yaml
+include: _template_yaml
 task: blimp_existential_there_quantifiers_2

lm_eval/tasks/blimp/existential_there_subject_raising.yaml

 # Generated by utils.py
 dataset_name: existential_there_subject_raising
-include: template_yaml
+include: _template_yaml
 task: blimp_existential_there_subject_raising

lm_eval/tasks/blimp/expletive_it_object_raising.yaml

 # Generated by utils.py
 dataset_name: expletive_it_object_raising
-include: template_yaml
+include: _template_yaml
 task: blimp_expletive_it_object_raising

lm_eval/tasks/blimp/generate_configs.py

@@ -80,7 +80,7 @@ def main() -> None:
                 f.write("# Generated by utils.py\n")
                 yaml.dump(
                     {
-                        "include": "template_yaml",
+                        "include": "_template_yaml",
                         "task": "blimp_" + task,
                         "dataset_name": task,
                     },

lm_eval/tasks/blimp/inchoative.yaml

 # Generated by utils.py
 dataset_name: inchoative
-include: template_yaml
+include: _template_yaml
 task: blimp_inchoative

lm_eval/tasks/blimp/intransitive.yaml

 # Generated by utils.py
 dataset_name: intransitive
-include: template_yaml
+include: _template_yaml
 task: blimp_intransitive

lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml

 # Generated by utils.py
 dataset_name: irregular_past_participle_adjectives
-include: template_yaml
+include: _template_yaml
 task: blimp_irregular_past_participle_adjectives

lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml

 # Generated by utils.py
 dataset_name: irregular_past_participle_verbs
-include: template_yaml
+include: _template_yaml
 task: blimp_irregular_past_participle_verbs

lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml

 # Generated by utils.py
 dataset_name: irregular_plural_subject_verb_agreement_1
-include: template_yaml
+include: _template_yaml
 task: blimp_irregular_plural_subject_verb_agreement_1

lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml

 # Generated by utils.py
 dataset_name: irregular_plural_subject_verb_agreement_2
-include: template_yaml
+include: _template_yaml
 task: blimp_irregular_plural_subject_verb_agreement_2

lm_eval/tasks/blimp/left_branch_island_echo_question.yaml

 # Generated by utils.py
 dataset_name: left_branch_island_echo_question
-include: template_yaml
+include: _template_yaml
 task: blimp_left_branch_island_echo_question

lm_eval/tasks/blimp/left_branch_island_simple_question.yaml

 # Generated by utils.py
 dataset_name: left_branch_island_simple_question
-include: template_yaml
+include: _template_yaml
 task: blimp_left_branch_island_simple_question

lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml

 # Generated by utils.py
 dataset_name: matrix_question_npi_licensor_present
-include: template_yaml
+include: _template_yaml
 task: blimp_matrix_question_npi_licensor_present