Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
8e4ff678
Commit
8e4ff678
authored
Oct 24, 2020
by
Anish Thite
Browse files
update lambada for new dataset class
parent
d4daf44c
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
6 additions
and
3 deletions
+6
-3
lm_eval/tasks/lambada.py
lm_eval/tasks/lambada.py
+6
-3
No files found.
lm_eval/tasks/lambada.py
View file @
8e4ff678
...
@@ -6,7 +6,8 @@ import ftfy
...
@@ -6,7 +6,8 @@ import ftfy
class
Lambada
(
Dataset
):
class
Lambada
(
Dataset
):
def
__init__
(
self
):
self
.
download
()
def
download
(
self
):
def
download
(
self
):
sh
(
"mkdir -p data/lambada"
)
sh
(
"mkdir -p data/lambada"
)
with
open
(
"data/lambada/lambada_test.json"
,
'w'
)
as
f
:
with
open
(
"data/lambada/lambada_test.json"
,
'w'
)
as
f
:
...
@@ -32,14 +33,16 @@ class Lambada(Dataset):
...
@@ -32,14 +33,16 @@ class Lambada(Dataset):
pass
pass
def
load_doc
(
self
,
myjson
):
def
load_doc
(
self
,
myjson
):
return
[
doc
[
'text'
]
for
doc
in
myjson
]
return
[
doc
for
doc
in
myjson
]
def
test_docs
(
self
):
def
test_docs
(
self
):
myjson
=
json
.
load
(
open
(
"data/lambada/lambada_test.json"
))
myjson
=
json
.
load
(
open
(
"data/lambada/lambada_test.json"
))
return
self
.
load_doc
(
myjson
)
return
self
.
load_doc
(
myjson
)
def
doc_to_text
(
self
,
doc
,
include_target
=
True
):
def
doc_to_text
(
self
,
doc
,
include_target
=
True
):
pass
#TODO: check if this is how OA does it
#label = doc[]
return
doc
def
evaluate
(
self
,
docs
,
lm
,
provide_description
,
num_fewshot
):
def
evaluate
(
self
,
docs
,
lm
,
provide_description
,
num_fewshot
):
pass
pass
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment