Skip to content

GitLab

Commit 3fbdfea1 authored by haileyschoelkopf's avatar haileyschoelkopf
Browse files

update regeneration script, bump bbh_cot_fewshot version

parent 7ec42165
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 3 deletions
lm_eval/tasks/bbh/_generate_configs.py
View file @ 3fbdfea1
...@@ -11,7 +11,6 @@ import datasets ...@@ -11,7 +11,6 @@ import datasets
from tqdm import tqdm from tqdm import tqdm
from lm_eval import utils from lm_eval import utils
from lm_eval.logger import eval_logger
def parse_args(): def parse_args():
...@@ -43,7 +42,7 @@ if __name__ == "__main__": ...@@ -43,7 +42,7 @@ if __name__ == "__main__":
f"https://raw.githubusercontent.com/suzgunmirac/BIG-Bench-Hard/main/cot-prompts/{task}.txt" f"https://raw.githubusercontent.com/suzgunmirac/BIG-Bench-Hard/main/cot-prompts/{task}.txt"
).content.decode("utf-8") ).content.decode("utf-8")
prompt = resp.split("\n-----\n")[-1] prompt = resp.split("\n-----\n")[-1]
description, *few_shot = prompt.split("\n\nQ:") description, *few_shot = prompt.split("\n\n")
prefix_doc_to_text = "" prefix_doc_to_text = ""
if args.fewshot: if args.fewshot:
...@@ -73,7 +72,7 @@ if __name__ == "__main__": ...@@ -73,7 +72,7 @@ if __name__ == "__main__":
} }
file_save_path = args.save_prefix_path + f"/{task}.yaml" file_save_path = args.save_prefix_path + f"/{task}.yaml"
eval_logger.info(f"Saving yaml for subset {task} to {file_save_path}") utils.eval_logger.info(f"Saving yaml for subset {task} to {file_save_path}")
with open(file_save_path, "w") as yaml_file: with open(file_save_path, "w") as yaml_file:
yaml.dump( yaml.dump(
yaml_dict, yaml_dict,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment