resolved merge conflict

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/ruin_names.yaml

@@ -2,4 +2,4 @@
 "description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n"
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_01_zeroshot_ruin_names"
-"process_docs": !function ../../utils.fix_ruin_names
\ No newline at end of file
+"process_docs": !function ../../utils.fix_ruin_names

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/sports_understanding.yaml

@@ -3,4 +3,4 @@
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_01_zeroshot_sports_understanding"
 "doc_to_target": target
-"doc_to_choice": ["yes", "no"]
\ No newline at end of file
+"doc_to_choice": ["yes", "no"]

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_01/zeroshot/web_of_lies.yaml

@@ -3,4 +3,4 @@
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_01_zeroshot_web_of_lies"
 "doc_to_target": target
-"doc_to_choice": ["Yes", "No"]
\ No newline at end of file
+"doc_to_choice": ["Yes", "No"]

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/_zeroshot_template_yaml

@@ -9,4 +9,4 @@ num_fewshot: 0
 metric_list:
   - metric: acc
   - metric: acc_norm
-  - metric: brier_score
\ No newline at end of file
+  - metric: brier_score

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/formal_fallacies.yaml

@@ -3,4 +3,4 @@
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_02_zeroshot_formal_fallacies"
 "doc_to_target": target
-"doc_to_choice": ["valid", "invalid"]
\ No newline at end of file
+"doc_to_choice": ["valid", "invalid"]

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/movie_recommendation.yaml

@@ -2,4 +2,4 @@
 "description": "Recommend movies similar to the given list of movies.\n\n"
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_02_zeroshot_movie_recommendation"
-"process_docs": !function ../../utils.fix_movie_recommendation
\ No newline at end of file
+"process_docs": !function ../../utils.fix_movie_recommendation

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/ruin_names.yaml

@@ -2,4 +2,4 @@
 "description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n"
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_02_zeroshot_ruin_names"
-"process_docs": !function ../../utils.fix_ruin_names
\ No newline at end of file
+"process_docs": !function ../../utils.fix_ruin_names

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/sports_understanding.yaml

@@ -3,4 +3,4 @@
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_02_zeroshot_sports_understanding"
 "doc_to_target": target
-"doc_to_choice": ["yes", "no"]
\ No newline at end of file
+"doc_to_choice": ["yes", "no"]

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_02/zeroshot/web_of_lies.yaml

@@ -3,4 +3,4 @@
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_02_zeroshot_web_of_lies"
 "doc_to_target": target
-"doc_to_choice": ["Yes", "No"]
\ No newline at end of file
+"doc_to_choice": ["Yes", "No"]

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/_zeroshot_template_yaml

@@ -9,4 +9,4 @@ num_fewshot: 0
 metric_list:
   - metric: acc
   - metric: acc_norm
-  - metric: brier_score
\ No newline at end of file
+  - metric: brier_score

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/formal_fallacies.yaml

@@ -3,4 +3,4 @@
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_03_zeroshot_formal_fallacies"
 "doc_to_target": target
-"doc_to_choice": ["valid", "invalid"]
\ No newline at end of file
+"doc_to_choice": ["valid", "invalid"]

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/movie_recommendation.yaml

@@ -2,4 +2,4 @@
 "description": "Recommend movies similar to the given list of movies.\n\n"
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_03_zeroshot_movie_recommendation"
-"process_docs": !function ../../utils.fix_movie_recommendation
\ No newline at end of file
+"process_docs": !function ../../utils.fix_movie_recommendation

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/ruin_names.yaml

@@ -2,4 +2,4 @@
 "description": "Select the humorous edit that 'ruins' the input movie or musical artist name.\n\n"
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_03_zeroshot_ruin_names"
-"process_docs": !function ../../utils.fix_ruin_names
\ No newline at end of file
+"process_docs": !function ../../utils.fix_ruin_names

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/sports_understanding.yaml

@@ -3,4 +3,4 @@
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_03_zeroshot_sports_understanding"
 "doc_to_target": target
-"doc_to_choice": ["yes", "no"]
\ No newline at end of file
+"doc_to_choice": ["yes", "no"]

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/style_03/zeroshot/web_of_lies.yaml

@@ -3,4 +3,4 @@
 "include": "_zeroshot_template_yaml"
 "task": "bbh_alt_pv_03_zeroshot_web_of_lies"
 "doc_to_target": target
-"doc_to_choice": ["Yes", "No"]
\ No newline at end of file
+"doc_to_choice": ["Yes", "No"]

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/styles.py

@@ -3,6 +3,7 @@ import string
 
 yes_no = ["Yes", "No"]
 
+
 def parse_choices(doc):
 
     input_text = doc["input"]
@@ -20,8 +21,9 @@ def parse_choices(doc):
     else:
         return []
 
+
 def styles_01(doc):
-    #Check for choices and remove them
+    # Check for choices and remove them
     choices = parse_choices(doc)
     if choices != []:
         doc_to_text = doc["input"].split("Options:")[0]
@@ -31,19 +33,23 @@ def styles_01(doc):
         doc_to_text = doc["input"]
     return doc_to_text
 
+
 def styles_02(doc):
-    #Check for choices and remove them
+    # Check for choices and remove them
     doc_to_text = styles_01(doc)
-    return "Q: "+doc_to_text+"\nA:"
+    return "Q: " + doc_to_text + "\nA:"
+
 
 def styles_03(doc):
-    #Check for choices and remove them
+    # Check for choices and remove them
     doc_to_text = styles_01(doc)
-    return "Question: "+doc_to_text+"\nAnswer:"
+    return "Question: " + doc_to_text + "\nAnswer:"
+
 
 def doc_to_choice(doc):
     return parse_choices(doc)
 
+
 def doc_to_target(doc):
     target = doc["target"]
     try:
@@ -61,4 +67,6 @@ def doc_to_target(doc):
         print(parse_choices(doc))
         print("Error")
         print(err)
-        import sys; sys.exit()
\ No newline at end of file
+        import sys
+
+        sys.exit()

lm_eval/tasks/bbh/alternative_worlds/prompt_variation/utils.py

 # For fixing line 163 in `movie_recommendation`
 
-def fix_movie_recommendation(data):
 
+def fix_movie_recommendation(data):
     def _fix(doc):
         if doc["target"] == "Monsters, Inc":
-            doc["input"] = "Find a movie similar to Minority Report, Shrek, Catch Me If You Can, Aladdin:\nOptions:\n(A) Monsters, Inc\n(B) Children of the Night\n(C) The Incredible Shrinking Man\n(D) Town & Country"
+            doc[
+                "input"
+            ] = "Find a movie similar to Minority Report, Shrek, Catch Me If You Can, Aladdin:\nOptions:\n(A) Monsters, Inc\n(B) Children of the Night\n(C) The Incredible Shrinking Man\n(D) Town & Country"
             doc["target"] = "(A)"
         return doc
 
     return data.map(_fix)
 
-def fix_ruin_names(data):
 
+def fix_ruin_names(data):
     def _fix(doc):
         if doc["target"] == "dearth, wind, & fire":
-            doc["input"] = "Which of the following is a humorous edit of this artist or movie name: 'earth, wind, & fire'?\nOptions:\n(A) eareth, wind, & fire\n(B) earth, bind, & fire\n(C) earthm wind, & fire\n(D) dearth, wind, & fire"
+            doc[
+                "input"
+            ] = "Which of the following is a humorous edit of this artist or movie name: 'earth, wind, & fire'?\nOptions:\n(A) eareth, wind, & fire\n(B) earth, bind, & fire\n(C) earthm wind, & fire\n(D) dearth, wind, & fire"
             doc["target"] = "(D)"
 
         elif doc["target"] == "rita, sue and bob poo":
-            doc["input"] = "Which of the following is a humorous edit of this artist or movie name: 'rita, sue and bob too'?\nOptions:\n(A) rita, sue and bob too\n(B) rita, sue and bob poo\n(C) rita, sue and box too\n(D) rita,y sue and bob too"
+            doc[
+                "input"
+            ] = "Which of the following is a humorous edit of this artist or movie name: 'rita, sue and bob too'?\nOptions:\n(A) rita, sue and bob too\n(B) rita, sue and bob poo\n(C) rita, sue and box too\n(D) rita,y sue and bob too"
             doc["target"] = "(B)"
         return doc
 
-    return data.map(_fix)
\ No newline at end of file
+    return data.map(_fix)

lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml

@@ -24,5 +24,6 @@ filter_list:
       - function: "regex"
         regex_pattern: "(?<=the answer is )(.*)(?=.)"
       - function: "take_first"
+num_fewshot: 0
 metadata:
   - version: 0.0

lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml

@@ -22,5 +22,6 @@ filter_list:
       - function: "regex"
         regex_pattern: "((?<=The answer is )(.*)(?=.)|(?<=the answer is )(.*)(?=.)|(?<=The answer: )(.*)(?=.)|(?<=The final answer: )(.*)(?=.))"
       - function: "take_first"
+num_fewshot: 0
 metadata:
   - version: 0

lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml

@@ -16,5 +16,6 @@ generation_kwargs:
     - "\n\n"
   do_sample: false
   temperature: 0.0
+num_fewshot: 0
 metadata:
   - version: 0