add prompt variation

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/logical_deduction_three_objects.yaml

+"dataset_name": "logical_deduction_three_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_logical_deduction_three_objects"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/movie_recommendation.yaml

+"dataset_name": "movie_recommendation"
+"description": "Recommend movies similar to the given list of movies.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_movie_recommendation"
+"process_docs": !function ../utils.fix_movie_recommendation
\ No newline at end of file

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/navigate.yaml

+"dataset_name": "navigate"
+"description": "Given a series of navigation instructions, determine whether one would end up back at the starting point.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_navigate"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/penguins_in_a_table.yaml

+"dataset_name": "penguins_in_a_table"
+"description": "Answer questions about a table of penguins and their attributes.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_penguins_in_a_table"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/reasoning_about_colored_objects.yaml

+"dataset_name": "reasoning_about_colored_objects"
+"description": "Answer extremely simple questions about the colors of objects on a surface.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_reasoning_about_colored_objects"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/ruin_names.yaml

+"dataset_name": "ruin_names"
+"description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_ruin_names"
+"process_docs": !function utils.fix_ruin_names
\ No newline at end of file

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/salient_translation_error_detection.yaml

+"dataset_name": "salient_translation_error_detection"
+"description": "Detect the type of error in an English translation of a German source sentence.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_salient_translation_error_detection"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/snarks.yaml

+"dataset_name": "snarks"
+"description": "Determine which of two sentences is sarcastic.\n\nAccording to Cambridge University Dictionary, sarcasm is \"the use of remarks that clearly mean the opposite of what they say, made in order to hurt someone's feelings or to criticize something in a humorous way.\" Sarcastic sentences often contain satirical or ironic utterances, hyperboles, ambivalent or witty remarks.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_snarks"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/sports_understanding.yaml

+"dataset_name": "sports_understanding"
+"description": "Determine whether an artificially constructed sentence relating to sports is plausible or not.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_sports_understanding"
+"doc_to_target": target
+"doc_to_choice": ["yes", "no"]
\ No newline at end of file

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/temporal_sequences.yaml

+"dataset_name": "temporal_sequences"
+"description": "Task description: Answer questions about which times certain events could have occurred.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_temporal_sequences"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_five_objects.yaml

+"dataset_name": "tracking_shuffled_objects_five_objects"
+"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_tracking_shuffled_objects_five_objects"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_seven_objects.yaml

+"dataset_name": "tracking_shuffled_objects_seven_objects"
+"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_tracking_shuffled_objects_seven_objects"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/tracking_shuffled_objects_three_objects.yaml

+"dataset_name": "tracking_shuffled_objects_three_objects"
+"description": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_tracking_shuffled_objects_three_objects"

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/web_of_lies.yaml

+"dataset_name": "web_of_lies"
+"description": "Evaluate a random boolean function expressed as a word problem.\n\n"
+"include": "_zeroshot_template_yaml"
+"task": "bbh_alt_pv_03_zeroshot_web_of_lies"
+"doc_to_target": target
+"doc_to_choice": ["Yes", "No"]
\ No newline at end of file

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/styles.py

+import re
+import string
+
+yes_no = ["Yes", "No"]
+
+def parse_choices(doc):
+
+    input_text = doc["input"]
+    choice_string = input_text.split("Options:")
+    if len(choice_string) == 2:
+        choice_string = choice_string[-1]
+        if ("- Yes" in choice_string) and ("- No" in choice_string):
+            choices = yes_no
+        else:
+            choices = [
+                c[4:].rstrip("\n")
+                for c in re.findall(r"\([A-Z]\) .*?\n|\([A-Z]\) .*?$", choice_string)
+            ]
+        return choices
+    else:
+        return []
+
+def styles_01(doc):
+    #Check for choices and remove them
+    choices = parse_choices(doc)
+    if choices != []:
+        doc_to_text = doc["input"].split("Options:")[0]
+        if doc_to_text[-1] in ["\n", " "]:
+            doc_to_text = doc_to_text[:-1]
+    else:
+        doc_to_text = doc["input"]
+    return doc_to_text
+
+def styles_02(doc):
+    #Check for choices and remove them
+    doc_to_text = styles_01(doc)
+    return "Q: "+doc_to_text+"\nA:"
+
+def styles_03(doc):
+    #Check for choices and remove them
+    doc_to_text = styles_01(doc)
+    return "Question: "+doc_to_text+"\nAnswer:"
+
+def doc_to_choice(doc):
+    return parse_choices(doc)
+
+def doc_to_target(doc):
+    target = doc["target"]
+    try:
+        if target in ["Yes", "No"]:
+            return yes_no.index(target)
+        else:
+            return string.ascii_uppercase.index(target[1:-1])
+        # else:
+        #     return parse_choices(doc).index(target)
+
+    except Exception as err:
+        print("Full Doc")
+        print(doc)
+        print("Choices")
+        print(parse_choices(doc))
+        print("Error")
+        print(err)
+        import sys; sys.exit()
\ No newline at end of file

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/utils.py

+# For fixing line 163 in `movie_recommendation`
+
+def fix_movie_recommendation(data):
+
+    def _fix(doc):
+        if doc["target"] == "Monsters, Inc":
+            doc["input"] = "Find a movie similar to Minority Report, Shrek, Catch Me If You Can, Aladdin:\nOptions:\n(A) Monsters, Inc\n(B) Children of the Night\n(C) The Incredible Shrinking Man\n(D) Town & Country"
+            doc["target"] = "(A)"
+        return doc
+
+    return data.map(_fix)
+
+def fix_ruin_names(data):
+
+    def _fix(doc):
+        if doc["target"] == "dearth, wind, & fire":
+            doc["input"] = "Which of the following is a humorous edit of this artist or movie name: 'earth, wind, & fire'?\nOptions:\n(A) eareth, wind, & fire\n(B) earth, bind, & fire\n(C) earthm wind, & fire\n(D) dearth, wind, & fire"
+            doc["target"] = "(D)"
+
+        elif doc["target"] == "rita, sue and bob poo":
+            doc["input"] = "Which of the following is a humorous edit of this artist or movie name: 'rita, sue and bob too'?\nOptions:\n(A) rita, sue and bob too\n(B) rita, sue and bob poo\n(C) rita, sue and box too\n(D) rita,y sue and bob too"
+            doc["target"] = "(B)"
+        return doc
+
+    return data.map(_fix)
\ No newline at end of file