Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
f4120e59
"vscode:/vscode.git/clone" did not exist on "cc56450c086216edfe09527f3f683bd057a0a865"
Commit
f4120e59
authored
Jan 21, 2021
by
Leo Gao
Browse files
Remove rewrite reminder comment from everything except SuperGLUE
parent
e31b4b31
Changes
18
Hide whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
2 additions
and
37 deletions
+2
-37
lm_eval/tasks/anli.py
lm_eval/tasks/anli.py
+0
-2
lm_eval/tasks/arc.py
lm_eval/tasks/arc.py
+0
-2
lm_eval/tasks/drop.py
lm_eval/tasks/drop.py
+0
-2
lm_eval/tasks/glue.py
lm_eval/tasks/glue.py
+0
-2
lm_eval/tasks/hellaswag.py
lm_eval/tasks/hellaswag.py
+0
-2
lm_eval/tasks/lambada.py
lm_eval/tasks/lambada.py
+0
-2
lm_eval/tasks/naturalqs.py
lm_eval/tasks/naturalqs.py
+0
-2
lm_eval/tasks/openbookqa.py
lm_eval/tasks/openbookqa.py
+2
-3
lm_eval/tasks/piqa.py
lm_eval/tasks/piqa.py
+0
-2
lm_eval/tasks/quac.py
lm_eval/tasks/quac.py
+0
-2
lm_eval/tasks/race.py
lm_eval/tasks/race.py
+0
-2
lm_eval/tasks/squad.py
lm_eval/tasks/squad.py
+0
-2
lm_eval/tasks/storycloze.py
lm_eval/tasks/storycloze.py
+0
-2
lm_eval/tasks/triviaqa.py
lm_eval/tasks/triviaqa.py
+0
-2
lm_eval/tasks/webqs.py
lm_eval/tasks/webqs.py
+0
-2
lm_eval/tasks/wikitext.py
lm_eval/tasks/wikitext.py
+0
-2
lm_eval/tasks/winogrande.py
lm_eval/tasks/winogrande.py
+0
-2
lm_eval/tasks/wsc273.py
lm_eval/tasks/wsc273.py
+0
-2
No files found.
lm_eval/tasks/anli.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
from
.
common
import
HFTask
from
.
common
import
HFTask
class
ANLIBase
(
HFTask
):
class
ANLIBase
(
HFTask
):
...
...
lm_eval/tasks/arc.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
from
.
common
import
HFTask
from
.
common
import
HFTask
class
ARCEasy
(
HFTask
):
class
ARCEasy
(
HFTask
):
...
...
lm_eval/tasks/drop.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
numpy
as
np
import
numpy
as
np
import
json
import
json
from
scipy.stats
import
pearsonr
,
spearmanr
from
scipy.stats
import
pearsonr
,
spearmanr
...
...
lm_eval/tasks/glue.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
numpy
as
np
import
numpy
as
np
from
lm_eval.base
import
rf
,
mean
,
f1_score
,
matthews_corrcoef
from
lm_eval.base
import
rf
,
mean
,
f1_score
,
matthews_corrcoef
from
scipy.stats
import
pearsonr
,
spearmanr
from
scipy.stats
import
pearsonr
,
spearmanr
...
...
lm_eval/tasks/hellaswag.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
numpy
as
np
import
numpy
as
np
from
scipy.stats
import
pearsonr
,
spearmanr
from
scipy.stats
import
pearsonr
,
spearmanr
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
...
...
lm_eval/tasks/lambada.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
from
lm_eval.base
import
Dataset
from
lm_eval.base
import
Dataset
from
lm_eval.utils
import
sh
from
lm_eval.utils
import
sh
import
json
import
json
...
...
lm_eval/tasks/naturalqs.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
from
.
common
import
HFTask
from
.
common
import
HFTask
from
itertools
import
islice
from
itertools
import
islice
...
...
lm_eval/tasks/openbookqa.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
numpy
as
np
import
numpy
as
np
from
scipy.stats
import
pearsonr
,
spearmanr
from
scipy.stats
import
pearsonr
,
spearmanr
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
...
@@ -34,7 +32,8 @@ class OpenBookQA(HFTask):
...
@@ -34,7 +32,8 @@ class OpenBookQA(HFTask):
return
self
.
data
[
"test"
]
return
self
.
data
[
"test"
]
def
fewshot_description
(
self
):
def
fewshot_description
(
self
):
return
"Text of the question prompt
\n
Text of the answer completion"
# TODO: figure out fewshot description
return
""
def
doc_to_text
(
self
,
doc
):
def
doc_to_text
(
self
,
doc
):
return
doc
[
'question_stem'
]
+
'
\n
'
return
doc
[
'question_stem'
]
+
'
\n
'
...
...
lm_eval/tasks/piqa.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
json
import
json
import
random
import
random
from
lm_eval.base
import
Dataset
from
lm_eval.base
import
Dataset
...
...
lm_eval/tasks/quac.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
json
import
json
import
random
import
random
import
os
import
os
...
...
lm_eval/tasks/race.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
from
.
common
import
HFTask
from
.
common
import
HFTask
from
..utils_stream
import
X
,
each
,
apply
,
join
,
filt
,
one
from
..utils_stream
import
X
,
each
,
apply
,
join
,
filt
,
one
import
collections
import
collections
...
...
lm_eval/tasks/squad.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
numpy
as
np
import
numpy
as
np
from
scipy.stats
import
pearsonr
,
spearmanr
from
scipy.stats
import
pearsonr
,
spearmanr
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
...
...
lm_eval/tasks/storycloze.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
json
import
json
import
random
import
random
from
lm_eval.base
import
Dataset
from
lm_eval.base
import
Dataset
...
...
lm_eval/tasks/triviaqa.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
json
import
json
import
random
import
random
from
lm_eval.base
import
Dataset
from
lm_eval.base
import
Dataset
...
...
lm_eval/tasks/webqs.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
from
.
common
import
HFTask
from
.
common
import
HFTask
class
WebQs
(
HFTask
):
class
WebQs
(
HFTask
):
...
...
lm_eval/tasks/wikitext.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
numpy
as
np
import
numpy
as
np
from
scipy.stats
import
pearsonr
,
spearmanr
from
scipy.stats
import
pearsonr
,
spearmanr
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
...
...
lm_eval/tasks/winogrande.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
numpy
as
np
import
numpy
as
np
from
scipy.stats
import
pearsonr
,
spearmanr
from
scipy.stats
import
pearsonr
,
spearmanr
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
from
sklearn.metrics
import
f1_score
,
matthews_corrcoef
...
...
lm_eval/tasks/wsc273.py
View file @
f4120e59
# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
import
json
import
json
import
random
import
random
import
os
import
os
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment