Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
ec951cb9
Commit
ec951cb9
authored
Sep 01, 2024
by
Baber
Browse files
fixup: gen_prefix can be None
parent
d5471cfa
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
9 additions
and
9 deletions
+9
-9
lm_eval/api/task.py
lm_eval/api/task.py
+9
-9
No files found.
lm_eval/api/task.py
View file @
ec951cb9
...
@@ -1060,6 +1060,8 @@ class ConfigurableTask(Task):
...
@@ -1060,6 +1060,8 @@ class ConfigurableTask(Task):
else
:
else
:
labeled_examples
=
""
labeled_examples
=
""
gen_prefix
=
""
if
not
self
.
config
.
gen_prefix
else
self
.
config
.
gen_prefix
# get task description
# get task description
if
description
:
=
self
.
config
.
description
:
if
description
:
=
self
.
config
.
description
:
description
=
utils
.
apply_template
(
self
.
config
.
description
,
doc
)
description
=
utils
.
apply_template
(
self
.
config
.
description
,
doc
)
...
@@ -1107,7 +1109,7 @@ class ConfigurableTask(Task):
...
@@ -1107,7 +1109,7 @@ class ConfigurableTask(Task):
labeled_examples
,
labeled_examples
,
example
,
example
,
fewshot_as_multiturn
,
fewshot_as_multiturn
,
self
.
config
.
gen_prefix
,
gen_prefix
,
)
)
# for loglikelihood create a list of questions with appended choices
# for loglikelihood create a list of questions with appended choices
elif
isinstance
(
example
,
list
):
elif
isinstance
(
example
,
list
):
...
@@ -1126,14 +1128,14 @@ class ConfigurableTask(Task):
...
@@ -1126,14 +1128,14 @@ class ConfigurableTask(Task):
labeled_examples
,
labeled_examples
,
choices
[
example
],
choices
[
example
],
fewshot_as_multiturn
,
fewshot_as_multiturn
,
self
.
config
.
gen_prefix
,
gen_prefix
,
)
)
else
:
else
:
self
.
append_target_question
(
self
.
append_target_question
(
labeled_examples
,
labeled_examples
,
str
(
example
),
str
(
example
),
fewshot_as_multiturn
,
fewshot_as_multiturn
,
self
.
config
.
gen_prefix
,
gen_prefix
,
)
)
# return lm.apply_chat_template(labeled_examples)
# return lm.apply_chat_template(labeled_examples)
return
chat_template
(
labeled_examples
)
return
chat_template
(
labeled_examples
)
...
@@ -1142,17 +1144,15 @@ class ConfigurableTask(Task):
...
@@ -1142,17 +1144,15 @@ class ConfigurableTask(Task):
# TODO<baber>: How to handle gen_prefix for multiple inputs?
# TODO<baber>: How to handle gen_prefix for multiple inputs?
return
labeled_examples
return
labeled_examples
if
isinstance
(
example
,
str
):
if
isinstance
(
example
,
str
):
return
labeled_examples
+
example
+
self
.
config
.
gen_prefix
return
labeled_examples
+
example
+
gen_prefix
elif
isinstance
(
example
,
list
):
elif
isinstance
(
example
,
list
):
return
[
return
[
labeled_examples
+
ex
+
gen_prefix
for
ex
in
example
]
labeled_examples
+
ex
+
self
.
config
.
gen_prefix
for
ex
in
example
]
elif
isinstance
(
example
,
int
):
elif
isinstance
(
example
,
int
):
if
self
.
config
.
doc_to_choice
is
not
None
:
if
self
.
config
.
doc_to_choice
is
not
None
:
choices
=
self
.
doc_to_choice
(
doc
)
choices
=
self
.
doc_to_choice
(
doc
)
return
labeled_examples
+
choices
[
example
]
+
self
.
config
.
gen_prefix
return
labeled_examples
+
choices
[
example
]
+
gen_prefix
else
:
else
:
return
labeled_examples
+
str
(
example
)
+
self
.
config
.
gen_prefix
return
labeled_examples
+
str
(
example
)
+
gen_prefix
def
apply_filters
(
self
):
def
apply_filters
(
self
):
"""Iterates over FilterEnsembles and applies them to instances"""
"""Iterates over FilterEnsembles and applies them to instances"""
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment