Commit e3b99f68 authored by Baber's avatar Baber
Browse files

Revert "add Question: Answer to niah"

This reverts commit f70ec454.
parent 611ae88f
...@@ -8,7 +8,7 @@ test_split: test ...@@ -8,7 +8,7 @@ test_split: test
download_dataset: !function niah_utils.niah_single_1 download_dataset: !function niah_utils.niah_single_1
doc_to_text: "{{input}}" doc_to_text: "{{input}}"
doc_to_target: "{{outputs[0]}}" doc_to_target: "{{outputs[0]}}"
gen_prefix: "Answer: {{gen_prefix}}" gen_prefix: "{{gen_prefix}}"
target_delimiter: " " target_delimiter: " "
process_results: !function common_utils.process_results process_results: !function common_utils.process_results
metric_list: metric_list:
......
...@@ -7,7 +7,7 @@ from lm_eval.tasks.ruler.common_utils import DEFAULT_SEQ_LENGTHS, get_tokenizer ...@@ -7,7 +7,7 @@ from lm_eval.tasks.ruler.common_utils import DEFAULT_SEQ_LENGTHS, get_tokenizer
from lm_eval.tasks.ruler.prepare_niah import generate_samples, get_haystack from lm_eval.tasks.ruler.prepare_niah import generate_samples, get_haystack
TEMPLATE = """Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n{context}\nQuestion: What are all the special magic {type_needle_v} for {query} mentioned in the provided text?""" TEMPLATE = """Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat are all the special magic {type_needle_v} for {query} mentioned in the provided text?"""
def download_dataset(df: Generator) -> dict[str, datasets.Dataset]: def download_dataset(df: Generator) -> dict[str, datasets.Dataset]:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment