Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
4849702e
Unverified
Commit
4849702e
authored
Jan 14, 2022
by
Leo Gao
Committed by
GitHub
Jan 14, 2022
Browse files
Merge pull request #256 from jon-tow/update-asdiv
Conform `asidv` to the new description api
parents
be40299e
caff4f10
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
8 additions
and
5 deletions
+8
-5
lm_eval/tasks/asdiv.py
lm_eval/tasks/asdiv.py
+8
-5
No files found.
lm_eval/tasks/asdiv.py
View file @
4849702e
...
@@ -31,7 +31,7 @@ class Asdiv(Task):
...
@@ -31,7 +31,7 @@ class Asdiv(Task):
def
download
(
self
):
def
download
(
self
):
if
self
.
DATASET_PATH
.
exists
():
if
self
.
DATASET_PATH
.
exists
():
return
return
Path
.
mkdir
(
self
.
DATASET_PATH
)
Path
.
mkdir
(
self
.
DATASET_PATH
,
parents
=
True
)
url
=
"https://github.com/chaochun/nlu-asdiv-dataset/archive/55790e5270bb91ccfa5053194b25732534696b50.zip"
url
=
"https://github.com/chaochun/nlu-asdiv-dataset/archive/55790e5270bb91ccfa5053194b25732534696b50.zip"
checksum
=
"8f1fe4f6d5f170ec1e24ab78c244153c14c568b1bb2b1dad0324e71f37939a2d"
checksum
=
"8f1fe4f6d5f170ec1e24ab78c244153c14c568b1bb2b1dad0324e71f37939a2d"
zip_path
=
self
.
DATASET_PATH
/
"55790e5270bb91ccfa5053194b25732534696b50.zip"
zip_path
=
self
.
DATASET_PATH
/
"55790e5270bb91ccfa5053194b25732534696b50.zip"
...
@@ -75,10 +75,14 @@ class Asdiv(Task):
...
@@ -75,10 +75,14 @@ class Asdiv(Task):
data_xml_path
=
self
.
DATASET_PATH
/
"nlu-asdiv-dataset-55790e5270bb91ccfa5053194b25732534696b50/dataset/ASDiv.xml"
data_xml_path
=
self
.
DATASET_PATH
/
"nlu-asdiv-dataset-55790e5270bb91ccfa5053194b25732534696b50/dataset/ASDiv.xml"
return
self
.
load_docs
(
data_xml_path
)
return
self
.
load_docs
(
data_xml_path
)
def
fewshot_context
(
self
,
doc
,
num_fewshot
,
provide_description
,
rnd
):
def
fewshot_context
(
self
,
doc
,
num_fewshot
,
provide_description
=
None
,
rnd
=
None
,
description
=
None
):
assert
num_fewshot
==
0
,
"ASDiv is intended only for the zero-shot setting."
assert
num_fewshot
==
0
,
"ASDiv is intended only for the zero-shot setting."
return
super
().
fewshot_context
(
doc
,
num_fewshot
,
provide_description
,
rnd
)
return
super
().
fewshot_context
(
doc
=
doc
,
num_fewshot
=
num_fewshot
,
rnd
=
rnd
,
description
=
description
)
def
fewshot_description
(
self
):
def
fewshot_description
(
self
):
# TODO: add solution-type and formula
# TODO: add solution-type and formula
...
@@ -115,4 +119,3 @@ class Asdiv(Task):
...
@@ -115,4 +119,3 @@ class Asdiv(Task):
return
{
return
{
'acc'
:
True
'acc'
:
True
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment