update

lm_eval/tasks/bbh/_generate_configs.py

@@ -13,18 +13,13 @@ from tqdm import tqdm
 from lm_eval import utils
 from lm_eval.logger import eval_logger
 
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--base_yaml_path", required=True)
-    parser.add_argument(
-        "--save_prefix_path", default="flan_zeroshot"
-    )
-    parser.add_argument(
-        "--cot", default=False
-    )
-    parser.add_argument(
-        "--fewshot", default=False
-    )
+    parser.add_argument("--save_prefix_path", default="flan_zeroshot")
+    parser.add_argument("--cot", default=False)
+    parser.add_argument("--fewshot", default=False)
     parser.add_argument("--task_prefix", default="")
     return parser.parse_args()
 
@@ -44,7 +39,9 @@ if __name__ == "__main__":
     dataset_path = "lukaemon/bbh"
     for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()):
 
-        resp = requests.get(f"https://raw.githubusercontent.com/suzgunmirac/BIG-Bench-Hard/main/cot-prompts/{task}.txt").content.decode('utf-8')
+        resp = requests.get(
+            f"https://raw.githubusercontent.com/suzgunmirac/BIG-Bench-Hard/main/cot-prompts/{task}.txt"
+        ).content.decode("utf-8")
         prompt = resp.split("\n-----\n")[-1]
         description, *few_shot = prompt.split("\n\nQ:")
 
@@ -54,13 +51,13 @@ if __name__ == "__main__":
                 prefix_doc_to_text = " ".join(few_shot)
             else:
                 for shot in few_shot:
-                    shot = "Q:"+shot
+                    shot = "Q:" + shot
                     try:
                         answer = answer_regex.search(shot)[0]
                     except:
                         print("task", task)
                         print(shot)
-                    example = shot.split("Let\'s think step by step.")[0]
+                    example = shot.split("Let's think step by step.")[0]
                     prefix_doc_to_text += f"{example}{answer}\n\n"
 
         doc_to_text = prefix_doc_to_text + base_doc_to_text
@@ -68,17 +65,20 @@ if __name__ == "__main__":
             doc_to_text = doc_to_text + " Let's think step by step.\n"
 
         yaml_dict = {
-                "include": base_yaml_name,
-                "task": f"bbh_{args.task_prefix}_{task}",
-                "dataset_name": task,
-                "description": description+"\n\n",
-                "doc_to_text": doc_to_text,
-            }
+            "include": base_yaml_name,
+            "task": f"bbh_{args.task_prefix}_{task}",
+            "dataset_name": task,
+            "description": description + "\n\n",
+            "doc_to_text": doc_to_text,
+        }
 
         file_save_path = args.save_prefix_path + f"/{task}.yaml"
         eval_logger.info(f"Saving yaml for subset {task} to {file_save_path}")
         with open(file_save_path, "w") as yaml_file:
-            yaml.dump(yaml_dict, yaml_file, width=float("inf"), allow_unicode=True, default_style='"')
-
-
-
+            yaml.dump(
+                yaml_dict,
+                yaml_file,
+                width=float("inf"),
+                allow_unicode=True,
+                default_style='"',
+            )

lm_eval/tasks/bbh/flan_cot_fewshot/_flan_cot_fewshot_template_yaml

@@ -19,4 +19,4 @@ filter_list:
     filter:
       - function: "regex"
         regex_pattern: "(?<=the answer is )(.*)(?=.)"
-      - function: "take_first"
\ No newline at end of file
+      - function: "take_first"

lm_eval/tasks/bbh/flan_cot_zeroshot/_flan_cot_zeroshot_template_yaml

@@ -19,4 +19,4 @@ filter_list:
     filter:
       - function: "regex"
         regex_pattern: "(?<=the answer is )(.*)(?=.)"
-      - function: "take_first"
\ No newline at end of file
+      - function: "take_first"