Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
7cbd0bba
Commit
7cbd0bba
authored
Jun 12, 2021
by
Leo Gao
Browse files
Look at all docs of task in test
parent
dc648d84
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
6 additions
and
6 deletions
+6
-6
tests/test_tasks.py
tests/test_tasks.py
+6
-6
No files found.
tests/test_tasks.py
View file @
7cbd0bba
...
...
@@ -28,9 +28,10 @@ def test_basic_interface(taskname, Task):
# (don't test train because it's slow)
task2
=
Task
()
limit
=
None
if
task
.
has_validation_docs
():
arr
=
list
(
islice
(
task
.
validation_docs
(),
100
))
arr2
=
list
(
islice
(
task2
.
validation_docs
(),
100
))
arr
=
list
(
islice
(
task
.
validation_docs
(),
limit
))
arr2
=
list
(
islice
(
task2
.
validation_docs
(),
limit
))
assert
arr
==
arr2
...
...
@@ -40,8 +41,8 @@ def test_basic_interface(taskname, Task):
assert
reqs
==
reqs2
if
task
.
has_test_docs
():
arr
=
list
(
islice
(
task
.
test_docs
(),
100
))
arr2
=
list
(
islice
(
task2
.
test_docs
(),
100
))
arr
=
list
(
islice
(
task
.
test_docs
(),
limit
))
arr2
=
list
(
islice
(
task2
.
test_docs
(),
limit
))
assert
arr
==
arr2
...
...
@@ -51,7 +52,6 @@ def test_basic_interface(taskname, Task):
assert
reqs
==
reqs2
@
pytest
.
mark
.
parametrize
(
"taskname,Task"
,
tasks
.
TASK_REGISTRY
.
items
())
def
test_documents_and_requests
(
taskname
,
Task
):
print
(
'Evaluating task'
,
taskname
)
...
...
@@ -59,7 +59,7 @@ def test_documents_and_requests(taskname, Task):
fns
=
[]
if
task
.
has_training_docs
():
fns
.
append
(
task
.
training_docs
)
if
task
.
has_validation_docs
():
fns
.
append
(
task
.
validation_docs
)
# test doc
e
might not have labels
# test doc might not have labels
#if task.has_test_docs(): fns.append(task.test_docs)
for
fn
in
fns
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment