Merge pull request #816 from EleutherAI/flan-benchmark

[Refactor] Flan benchmark

lm_eval/tasks/bbh/flan_zeroshot/hyperbaton.yaml

+"dataset_name": "hyperbaton"
+"description": "Order adjectives correctly in English sentences.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_hyperbaton"

lm_eval/tasks/bbh/flan_zeroshot/logical_deduction_five_objects.yaml

+"dataset_name": "logical_deduction_five_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_logical_deduction_five_objects"

lm_eval/tasks/bbh/flan_zeroshot/logical_deduction_seven_objects.yaml

+"dataset_name": "logical_deduction_seven_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_logical_deduction_seven_objects"

lm_eval/tasks/bbh/flan_zeroshot/logical_deduction_three_objects.yaml

+"dataset_name": "logical_deduction_three_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_logical_deduction_three_objects"

lm_eval/tasks/bbh/flan_zeroshot/movie_recommendation.yaml

+"dataset_name": "movie_recommendation"
+"description": "Recommend movies similar to the given list of movies.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_movie_recommendation"

lm_eval/tasks/bbh/flan_zeroshot/multistep_arithmetic_two.yaml

+"dataset_name": "multistep_arithmetic_two"
+"description": "Solve multi-step arithmetic problems.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_multistep_arithmetic_two"

lm_eval/tasks/bbh/flan_zeroshot/navigate.yaml

+"dataset_name": "navigate"
+"description": "Given a series of navigation instructions, determine whether one would end up back at the starting point.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_navigate"

lm_eval/tasks/bbh/flan_zeroshot/object_counting.yaml

+"dataset_name": "object_counting"
+"description": "Questions that involve enumerating objects and asking the model to count them.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_object_counting"

lm_eval/tasks/bbh/flan_zeroshot/penguins_in_a_table.yaml

+"dataset_name": "penguins_in_a_table"
+"description": "Answer questions about a table of penguins and their attributes.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_penguins_in_a_table"

lm_eval/tasks/bbh/flan_zeroshot/reasoning_about_colored_objects.yaml

+"dataset_name": "reasoning_about_colored_objects"
+"description": "Answer extremely simple questions about the colors of objects on a surface.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_reasoning_about_colored_objects"

lm_eval/tasks/bbh/flan_zeroshot/ruin_names.yaml

+"dataset_name": "ruin_names"
+"description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_ruin_names"

lm_eval/tasks/bbh/flan_zeroshot/salient_translation_error_detection.yaml

+"dataset_name": "salient_translation_error_detection"
+"description": "Detect the type of error in an English translation of a German source sentence.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_salient_translation_error_detection"

lm_eval/tasks/bbh/flan_zeroshot/snarks.yaml

+"dataset_name": "snarks"
+"description": "Determine which of two sentences is sarcastic.\n\nAccording to Cambridge University Dictionary, sarcasm is \"the use of remarks that clearly mean the opposite of what they say, made in order to hurt someone's feelings or to criticize something in a humorous way.\" Sarcastic sentences often contain satirical or ironic utterances, hyperboles, ambivalent or witty remarks.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_snarks"

lm_eval/tasks/bbh/flan_zeroshot/sports_understanding.yaml

+"dataset_name": "sports_understanding"
+"description": "Determine whether an artificially constructed sentence relating to sports is plausible or not.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_sports_understanding"

lm_eval/tasks/bbh/flan_zeroshot/temporal_sequences.yaml

+"dataset_name": "temporal_sequences"
+"description": "Task description: Answer questions about which times certain events could have occurred.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_temporal_sequences"

lm_eval/tasks/bbh/flan_zeroshot/tracking_shuffled_objects_five_objects.yaml

+"dataset_name": "tracking_shuffled_objects_five_objects"
+"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_tracking_shuffled_objects_five_objects"

lm_eval/tasks/bbh/flan_zeroshot/tracking_shuffled_objects_seven_objects.yaml

+"dataset_name": "tracking_shuffled_objects_seven_objects"
+"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_tracking_shuffled_objects_seven_objects"

lm_eval/tasks/bbh/flan_zeroshot/tracking_shuffled_objects_three_objects.yaml

+"dataset_name": "tracking_shuffled_objects_three_objects"
+"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_tracking_shuffled_objects_three_objects"

lm_eval/tasks/bbh/flan_zeroshot/web_of_lies.yaml

+"dataset_name": "web_of_lies"
+"description": "Evaluate a random boolean function expressed as a word problem.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_web_of_lies"

lm_eval/tasks/bbh/flan_zeroshot/word_sorting.yaml

+"dataset_name": "word_sorting"
+"description": "Sort a list of words.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_word_sorting"