resolved merge conflict

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/styles.py

@@ -3,6 +3,7 @@ import string
 
 yes_no = ["Yes", "No"]
 
+
 def parse_choices(doc):
 
     input_text = doc["input"]
@@ -20,8 +21,9 @@ def parse_choices(doc):
     else:
         return []
 
+
 def styles_01(doc):
-    #Check for choices and remove them
+    # Check for choices and remove them
     choices = parse_choices(doc)
     if choices != []:
         doc_to_text = doc["input"].split("Options:")[0]
@@ -31,19 +33,23 @@ def styles_01(doc):
         doc_to_text = doc["input"]
     return doc_to_text
 
+
 def styles_02(doc):
-    #Check for choices and remove them
+    # Check for choices and remove them
     doc_to_text = styles_01(doc)
-    return "Q: "+doc_to_text+"\nA:"
+    return "Q: " + doc_to_text + "\nA:"
+
 
 def styles_03(doc):
-    #Check for choices and remove them
+    # Check for choices and remove them
     doc_to_text = styles_01(doc)
-    return "Question: "+doc_to_text+"\nAnswer:"
+    return "Question: " + doc_to_text + "\nAnswer:"
+
 
 def doc_to_choice(doc):
     return parse_choices(doc)
 
+
 def doc_to_target(doc):
     target = doc["target"]
     try:
@@ -61,4 +67,6 @@ def doc_to_target(doc):
         print(parse_choices(doc))
         print("Error")
         print(err)
-        import sys; sys.exit()
\ No newline at end of file
+        import sys
+
+        sys.exit()

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/utils.py

 # For fixing line 163 in `movie_recommendation`
 
-def fix_movie_recommendation(data):
 
+def fix_movie_recommendation(data):
     def _fix(doc):
         if doc["target"] == "Monsters, Inc":
-            doc["input"] = "Find a movie similar to Minority Report, Shrek, Catch Me If You Can, Aladdin:\nOptions:\n(A) Monsters, Inc\n(B) Children of the Night\n(C) The Incredible Shrinking Man\n(D) Town & Country"
+            doc[
+                "input"
+            ] = "Find a movie similar to Minority Report, Shrek, Catch Me If You Can, Aladdin:\nOptions:\n(A) Monsters, Inc\n(B) Children of the Night\n(C) The Incredible Shrinking Man\n(D) Town & Country"
             doc["target"] = "(A)"
         return doc
 
     return data.map(_fix)
 
-def fix_ruin_names(data):
 
+def fix_ruin_names(data):
     def _fix(doc):
         if doc["target"] == "dearth, wind, & fire":
-            doc["input"] = "Which of the following is a humorous edit of this artist or movie name: 'earth, wind, & fire'?\nOptions:\n(A) eareth, wind, & fire\n(B) earth, bind, & fire\n(C) earthm wind, & fire\n(D) dearth, wind, & fire"
+            doc[
+                "input"
+            ] = "Which of the following is a humorous edit of this artist or movie name: 'earth, wind, & fire'?\nOptions:\n(A) eareth, wind, & fire\n(B) earth, bind, & fire\n(C) earthm wind, & fire\n(D) dearth, wind, & fire"
             doc["target"] = "(D)"
 
         elif doc["target"] == "rita, sue and bob poo":
-            doc["input"] = "Which of the following is a humorous edit of this artist or movie name: 'rita, sue and bob too'?\nOptions:\n(A) rita, sue and bob too\n(B) rita, sue and bob poo\n(C) rita, sue and box too\n(D) rita,y sue and bob too"
+            doc[
+                "input"
+            ] = "Which of the following is a humorous edit of this artist or movie name: 'rita, sue and bob too'?\nOptions:\n(A) rita, sue and bob too\n(B) rita, sue and bob poo\n(C) rita, sue and box too\n(D) rita,y sue and bob too"
             doc["target"] = "(B)"
         return doc
 

lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml

@@ -24,5 +24,6 @@ filter_list:
       - function: "regex"
         regex_pattern: "(?<=the answer is )(.*)(?=.)"
       - function: "take_first"
+num_fewshot: 0
 metadata:
   - version: 0.0

lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml

@@ -22,5 +22,6 @@ filter_list:
       - function: "regex"
         regex_pattern: "((?<=The answer is )(.*)(?=.)|(?<=the answer is )(.*)(?=.)|(?<=The answer: )(.*)(?=.)|(?<=The final answer: )(.*)(?=.))"
       - function: "take_first"
+num_fewshot: 0
 metadata:
   - version: 0

lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml

@@ -16,5 +16,6 @@ generation_kwargs:
     - "\n\n"
   do_sample: false
   temperature: 0.0
+num_fewshot: 0
 metadata:
   - version: 0