Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
79878d13
Unverified
Commit
79878d13
authored
Feb 06, 2021
by
Leo Gao
Committed by
GitHub
Feb 06, 2021
Browse files
Remove merge conflict
parent
ab41f089
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
0 additions
and
71 deletions
+0
-71
lm_eval/tasks/pubmedqa.py
lm_eval/tasks/pubmedqa.py
+0
-71
No files found.
lm_eval/tasks/pubmedqa.py
View file @
79878d13
...
@@ -71,74 +71,3 @@ class Pubmed_QA(HFTask):
...
@@ -71,74 +71,3 @@ class Pubmed_QA(HFTask):
return
{
return
{
"acc"
:
True
"acc"
:
True
}
}
<<<<<<<
HEAD
=======
def
test_docs
(
self
):
if
self
.
has_test_docs
():
# HF is labelled as train but its really just for testing
return
self
.
data
[
"train"
]
class
SciQ
(
MultipleChoiceTask
):
def
download
(
self
):
if
not
os
.
path
.
exists
(
'data/sciq'
):
os
.
mkdir
(
'data/sciq'
)
sh
((
"wget https://ai2-public-datasets.s3.amazonaws.com/sciq/SciQ.zip -O data/sciq/SciQ.zip"
))
with
zipfile
.
ZipFile
(
"data/sciq/SciQ.zip"
,
"r"
)
as
zf
:
zf
.
extractall
(
"data/sciq/"
)
def
has_training_docs
(
self
):
return
True
def
has_validation_docs
(
self
):
return
True
def
has_test_docs
(
self
):
return
True
def
_convert_standard
(
self
,
doc
):
choices
=
[
doc
[
"distractor1"
],
doc
[
"distractor2"
],
doc
[
"distractor3"
],
doc
[
"correct_answer"
],
]
src
=
doc
[
'support'
]
out_doc
=
{
"source"
:
src
,
"query"
:
doc
[
'question'
],
"choices"
:
choices
,
"gold"
:
3
,
}
return
out_doc
def
load_docs
(
self
,
textfilename
):
with
open
(
textfilename
,
'r'
)
as
j
:
docs
=
json
.
loads
(
j
.
read
())
for
record
in
docs
:
yield
self
.
_convert_standard
(
record
)
def
fewshot_description
(
self
):
# Average ctx length in labelled dataset is 238.9
# 2 few-shot exmamples pushes it beyond context window
return
""
def
training_docs
(
self
):
return
self
.
load_docs
(
"data/sciq/SciQ dataset-2 3/train.json"
)
def
validation_docs
(
self
):
return
self
.
load_docs
(
"data/sciq/SciQ dataset-2 3/valid.json"
)
def
test_docs
(
self
):
return
self
.
load_docs
(
"data/sciq/SciQ dataset-2 3/test.json"
)
def
doc_to_text
(
self
,
doc
):
return
"{}
\n
{}"
.
format
(
doc
[
"source"
],
doc
[
"query"
])
class
EmrQA
():
def
load_docs
(
self
,
textfilename
):
pass
>>>>>>>
30
c2fe23657981d8a6155da1bd8e8098487b771a
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment