Merge pull request #816 from EleutherAI/flan-benchmark

[Refactor] Flan benchmark

lm_eval/tasks/bbh/flan_cot_zeroshot/disambiguation_qa.yaml

+"dataset_name": "disambiguation_qa"
+"description": "Clarify the meaning of sentences with ambiguous pronouns.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_disambiguation_qa"

lm_eval/tasks/bbh/flan_cot_zeroshot/dyck_languages.yaml

+"dataset_name": "dyck_languages"
+"description": "Correctly close a Dyck-n word.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_dyck_languages"

lm_eval/tasks/bbh/flan_cot_zeroshot/formal_fallacies.yaml

+"dataset_name": "formal_fallacies"
+"description": "Distinguish deductively valid arguments from formal fallacies.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_formal_fallacies"

lm_eval/tasks/bbh/flan_cot_zeroshot/geometric_shapes.yaml

+"dataset_name": "geometric_shapes"
+"description": "Name geometric shapes from their SVG paths.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_geometric_shapes"

lm_eval/tasks/bbh/flan_cot_zeroshot/hyperbaton.yaml

+"dataset_name": "hyperbaton"
+"description": "Order adjectives correctly in English sentences.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_hyperbaton"

lm_eval/tasks/bbh/flan_cot_zeroshot/logical_deduction_five_objects.yaml

+"dataset_name": "logical_deduction_five_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_logical_deduction_five_objects"

lm_eval/tasks/bbh/flan_cot_zeroshot/logical_deduction_seven_objects.yaml

+"dataset_name": "logical_deduction_seven_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_logical_deduction_seven_objects"

lm_eval/tasks/bbh/flan_cot_zeroshot/logical_deduction_three_objects.yaml

+"dataset_name": "logical_deduction_three_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_logical_deduction_three_objects"

lm_eval/tasks/bbh/flan_cot_zeroshot/movie_recommendation.yaml

+"dataset_name": "movie_recommendation"
+"description": "Recommend movies similar to the given list of movies.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_movie_recommendation"

lm_eval/tasks/bbh/flan_cot_zeroshot/multistep_arithmetic_two.yaml

+"dataset_name": "multistep_arithmetic_two"
+"description": "Solve multi-step arithmetic problems.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_multistep_arithmetic_two"

lm_eval/tasks/bbh/flan_cot_zeroshot/navigate.yaml

+"dataset_name": "navigate"
+"description": "Given a series of navigation instructions, determine whether one would end up back at the starting point.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_navigate"

lm_eval/tasks/bbh/flan_cot_zeroshot/object_counting.yaml

+"dataset_name": "object_counting"
+"description": "Questions that involve enumerating objects and asking the model to count them.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_object_counting"

lm_eval/tasks/bbh/flan_cot_zeroshot/penguins_in_a_table.yaml

+"dataset_name": "penguins_in_a_table"
+"description": "Answer questions about a table of penguins and their attributes.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_penguins_in_a_table"

lm_eval/tasks/bbh/flan_cot_zeroshot/reasoning_about_colored_objects.yaml

+"dataset_name": "reasoning_about_colored_objects"
+"description": "Answer extremely simple questions about the colors of objects on a surface.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_reasoning_about_colored_objects"

lm_eval/tasks/bbh/flan_cot_zeroshot/ruin_names.yaml

+"dataset_name": "ruin_names"
+"description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_ruin_names"

lm_eval/tasks/bbh/flan_cot_zeroshot/salient_translation_error_detection.yaml

+"dataset_name": "salient_translation_error_detection"
+"description": "Detect the type of error in an English translation of a German source sentence.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_salient_translation_error_detection"

lm_eval/tasks/bbh/flan_cot_zeroshot/snarks.yaml

+"dataset_name": "snarks"
+"description": "Determine which of two sentences is sarcastic.\n\nAccording to Cambridge University Dictionary, sarcasm is \"the use of remarks that clearly mean the opposite of what they say, made in order to hurt someone's feelings or to criticize something in a humorous way.\" Sarcastic sentences often contain satirical or ironic utterances, hyperboles, ambivalent or witty remarks.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_snarks"

lm_eval/tasks/bbh/flan_cot_zeroshot/sports_understanding.yaml

+"dataset_name": "sports_understanding"
+"description": "Determine whether an artificially constructed sentence relating to sports is plausible or not.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_sports_understanding"

lm_eval/tasks/bbh/flan_cot_zeroshot/temporal_sequences.yaml

+"dataset_name": "temporal_sequences"
+"description": "Task description: Answer questions about which times certain events could have occurred.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_temporal_sequences"

lm_eval/tasks/bbh/flan_cot_zeroshot/tracking_shuffled_objects_five_objects.yaml

+"dataset_name": "tracking_shuffled_objects_five_objects"
+"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n"
+"doc_to_text": "Q: {{input}}\nA: Let's think step by step.\n"
+"include": "_flan_cot_zeroshot_template_yaml"
+"task": "bbh_flan_cot_zeroshot_tracking_shuffled_objects_five_objects"