Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
6cc21f4b
Commit
6cc21f4b
authored
Oct 05, 2020
by
Leo Gao
Browse files
Fix include_target
parent
12852d5c
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
6 additions
and
6 deletions
+6
-6
lm_eval/tasks/arc.py
lm_eval/tasks/arc.py
+1
-2
lm_eval/tasks/race.py
lm_eval/tasks/race.py
+5
-4
No files found.
lm_eval/tasks/arc.py
View file @
6cc21f4b
...
@@ -18,9 +18,8 @@ class ARCEasy(HFNLPTask):
...
@@ -18,9 +18,8 @@ class ARCEasy(HFNLPTask):
return
""
return
""
def
doc_to_text
(
self
,
doc
,
include_target
=
True
):
def
doc_to_text
(
self
,
doc
,
include_target
=
True
):
print
(
doc
)
q
=
"Question: "
+
doc
[
'question'
]
+
'
\n
'
q
=
"Question: "
+
doc
[
'question'
]
+
'
\n
'
a
=
"Answer:"
+
(
" "
+
doc
[
'choices'
][
'text'
][
doc
[
'choices'
][
'label'
].
index
(
doc
[
'answerKey'
])])
if
include_target
else
""
a
=
"Answer:"
+
(
(
" "
+
doc
[
'choices'
][
'text'
][
doc
[
'choices'
][
'label'
].
index
(
doc
[
'answerKey'
])])
if
include_target
else
""
)
return
q
+
a
return
q
+
a
def
evaluate
(
self
,
docs
,
lm
,
provide_description
,
num_fewshot
):
def
evaluate
(
self
,
docs
,
lm
,
provide_description
,
num_fewshot
):
...
...
lm_eval/tasks/race.py
View file @
6cc21f4b
...
@@ -55,12 +55,13 @@ class RACE(HFNLPTask):
...
@@ -55,12 +55,13 @@ class RACE(HFNLPTask):
return
""
return
""
def
doc_to_text
(
self
,
doc
,
include_target
=
True
):
def
doc_to_text
(
self
,
doc
,
include_target
=
True
):
print
(
doc
)
r
=
"Article:
\n
"
+
doc
[
'article'
]
+
'
\n\n
'
r
=
"Article:
\n
"
+
doc
[
'article'
]
+
'
\n\n
'
r
+=
doc
[
'problems'
]
>>
each
(
r
+=
doc
[
'problems'
]
>>
apply
(
enumerate
)
>>
each
(
lambda
x
:
'Q: '
+
x
[
'question'
]
+
'
\n\n
A: '
+
x
[
'options'
][[
'A'
,
'B'
,
'C'
,
'D'
].
index
(
x
[
'answer'
])])
\
lambda
x
:
'Q: '
+
x
[
1
][
'question'
]
+
'
\n\n
A:'
>>
join
(
'
\n\n
'
)
+
((
' '
+
x
[
1
][
'options'
][[
'A'
,
'B'
,
'C'
,
'D'
].
index
(
x
[
1
][
'answer'
])])
\
if
x
[
0
]
!=
len
(
doc
[
'problems'
])
-
1
or
include_target
else
''
))
\
>>
join
(
'
\n\n
'
)
return
r
return
r
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment