Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
32612dc6
"vscode:/vscode.git/clone" did not exist on "fd67924ca171d7f5a51c12188a6f52e2d0baa994"
Commit
32612dc6
authored
Jan 13, 2023
by
Albert Jiang
Browse files
fix a bug with greedy eval
parent
3149b6ac
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
3 additions
and
2 deletions
+3
-2
lm_eval/base.py
lm_eval/base.py
+1
-1
lm_eval/tasks/hendrycks_math.py
lm_eval/tasks/hendrycks_math.py
+2
-1
No files found.
lm_eval/base.py
View file @
32612dc6
...
@@ -383,7 +383,7 @@ class BaseLM(LM):
...
@@ -383,7 +383,7 @@ class BaseLM(LM):
# multiple tokens or that span multiple tokens correctly
# multiple tokens or that span multiple tokens correctly
# TODO: extract to TokenizedLM?
# TODO: extract to TokenizedLM?
return
self
.
generate
(
requests
)
return
self
.
generate
(
requests
)
[
0
]
class
Task
(
abc
.
ABC
):
class
Task
(
abc
.
ABC
):
...
...
lm_eval/tasks/hendrycks_math.py
View file @
32612dc6
...
@@ -118,13 +118,14 @@ class Math(Task):
...
@@ -118,13 +118,14 @@ class Math(Task):
def
process_results
(
self
,
doc
,
results
,
description
=
""
):
def
process_results
(
self
,
doc
,
results
,
description
=
""
):
retval
=
0
retval
=
0
assert
isinstance
(
description
,
str
)
if
description
==
""
:
if
description
==
""
:
answer
=
self
.
get_pure_answer
(
results
[
0
])
answer
=
self
.
get_pure_answer
(
results
[
0
])
elif
self
.
MAJORITY_VOTING
in
self
.
parse_description
(
description
):
elif
self
.
MAJORITY_VOTING
in
self
.
parse_description
(
description
):
answer
=
self
.
majority_vote
(
results
[
0
])
answer
=
self
.
majority_vote
(
results
[
0
])
else
:
else
:
raise
AssertionError
raise
AssertionError
print
(
answer
)
if
self
.
is_equiv
(
if
self
.
is_equiv
(
answer
,
self
.
remove_boxed
(
self
.
last_boxed_only_string
(
doc
[
"solution"
]))
answer
,
self
.
remove_boxed
(
self
.
last_boxed_only_string
(
doc
[
"solution"
]))
):
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment