Skip to content

GitLab

Unverified Commit 867413f8 authored by Baber Abbasi's avatar Baber Abbasi Committed by GitHub
Browse files

Nits (#2500) 

* fix test task

* dont call lm.chat_template each time
parent 62b4364d
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 2 deletions
lm_eval/evaluator.py
View file @ 867413f8
...@@ -294,7 +294,9 @@ def simple_evaluate( ...@@ -294,7 +294,9 @@ def simple_evaluate(
model_source=model, model_source=model,
model_args=model_args, model_args=model_args,
system_instruction=system_instruction, system_instruction=system_instruction,
chat_template=lm.chat_template(apply_chat_template), chat_template=lm.chat_template(apply_chat_template)
if apply_chat_template
else None,
fewshot_as_multiturn=fewshot_as_multiturn, fewshot_as_multiturn=fewshot_as_multiturn,
) )
......
tests/test_tasks.py
View file @ 867413f8
...@@ -15,7 +15,7 @@ datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True ...@@ -15,7 +15,7 @@ datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True
os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["TOKENIZERS_PARALLELISM"] = "false"
task_manager = tasks.TaskManager() task_manager = tasks.TaskManager()
# Default Task # Default Task
TASKS = ["metabench_winogrande"] TASKS = ["arc_easy"]
def task_class(): def task_class():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment