Skip to content

GitLab

Unverified Commit a35eb973 authored by Slim Frikha's avatar Slim Frikha Committed by GitHub
Browse files

feat(scrolls): delete chat_template from kwargs (#3267)

parent 5ac7cdf8
Hide whitespace changes
Inline Side-by-side
Showing with 12 additions and 8 deletions
lm_eval/tasks/scrolls/task.py
View file @ a35eb973
...@@ -256,8 +256,9 @@ class _SCROLLSMultipleChoiceTask(_SCROLLSTask): ...@@ -256,8 +256,9 @@ class _SCROLLSMultipleChoiceTask(_SCROLLSTask):
"em": acc_norm * 100.0, "em": acc_norm * 100.0,
} }
def construct_requests(self, doc, ctx, **kwargs): def construct_requests(
apply_chat_template = kwargs.pop("apply_chat_template", False) self, doc, ctx, chat_template=None, apply_chat_template=False, **kwargs
):
request_list = [ request_list = [
Instance( Instance(
request_type="loglikelihood", request_type="loglikelihood",
...@@ -291,8 +292,9 @@ class _SCROLLSSummaryTask(_SCROLLSTask): ...@@ -291,8 +292,9 @@ class _SCROLLSSummaryTask(_SCROLLSTask):
"rougeL": (results[0], doc["outputs"]), "rougeL": (results[0], doc["outputs"]),
} }
def construct_requests(self, doc, ctx, **kwargs): def construct_requests(
kwargs.pop("apply_chat_template", False) self, doc, ctx, chat_template=None, apply_chat_template=False, **kwargs
):
return Instance( return Instance(
request_type="generate_until", request_type="generate_until",
doc=doc, doc=doc,
...@@ -334,8 +336,9 @@ class Qasper(_SCROLLSTask): ...@@ -334,8 +336,9 @@ class Qasper(_SCROLLSTask):
prediction = results[0] prediction = results[0]
return {"f1": (prediction, doc["outputs"])} return {"f1": (prediction, doc["outputs"])}
def construct_requests(self, doc, ctx, **kwargs): def construct_requests(
apply_chat_template = kwargs.pop("apply_chat_template", False) self, doc, ctx, chat_template=None, apply_chat_template=False, **kwargs
):
if doc["is_yes_no"]: if doc["is_yes_no"]:
return [ return [
Instance( Instance(
...@@ -416,8 +419,9 @@ class NarrativeQA(_SCROLLSTask): ...@@ -416,8 +419,9 @@ class NarrativeQA(_SCROLLSTask):
def process_results(self, doc, results): def process_results(self, doc, results):
return {"f1": (results[0], doc["outputs"])} return {"f1": (results[0], doc["outputs"])}
def construct_requests(self, doc, ctx, **kwargs): def construct_requests(
kwargs.pop("apply_chat_template", False) self, doc, ctx, chat_template=None, apply_chat_template=False, **kwargs
):
return Instance( return Instance(
request_type="generate_until", request_type="generate_until",
doc=doc, doc=doc,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment