Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
6d7a24b2
Commit
6d7a24b2
authored
Jun 05, 2023
by
lintangsutawika
Browse files
fixed typo
parent
82786023
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
23 additions
and
23 deletions
+23
-23
lm_eval/api/task.py
lm_eval/api/task.py
+23
-23
No files found.
lm_eval/api/task.py
View file @
6d7a24b2
...
@@ -486,29 +486,7 @@ class ConfigurableTask(Task):
...
@@ -486,29 +486,7 @@ class ConfigurableTask(Task):
self
.
_aggregation_list
=
{}
self
.
_aggregation_list
=
{}
self
.
_higher_is_better
=
{}
self
.
_higher_is_better
=
{}
if
self
.
_config
.
output_type
!=
"greedy_util"
:
if
self
.
_config
.
output_type
==
"greedy_until"
:
eval_logger
.
warning
(
f
"Output Type set as
{
self
.
_config
.
output_type
}
which does not use metric_list"
"metric list will be unused."
)
if
self
.
_config
.
output_type
==
"loglikelihood"
:
metric_list
=
[
"perplexity"
,
"acc"
]
elif
self
.
_config
.
output_type
==
"loglikelihood_rolling"
:
metric_list
=
[
"word_perplexity"
,
"byte_perplexity"
,
"bits_per_byte"
,
]
elif
self
.
_config
.
output_type
==
"multiple_choice"
:
metric_list
=
[
"acc"
,
"acc_norm"
]
for
metric_name
in
metric_list
:
self
.
_aggregation_list
[
metric_name
]
=
AGGREGATION_REGISTRY
[
"mean"
]
self
.
_higher_is_better
[
metric_name
]
=
HIGHER_IS_BETTER_REGISTRY
[
metric_name
]
else
:
for
metric_config
in
self
.
_config
.
metric_list
:
for
metric_config
in
self
.
_config
.
metric_list
:
metric_name
=
metric_config
[
"metric"
]
metric_name
=
metric_config
[
"metric"
]
...
@@ -543,6 +521,28 @@ class ConfigurableTask(Task):
...
@@ -543,6 +521,28 @@ class ConfigurableTask(Task):
),
),
"Please check https://huggingface.co/evaluate-metric"
,
"Please check https://huggingface.co/evaluate-metric"
,
)
)
else
:
eval_logger
.
warning
(
f
"Output Type set as
{
self
.
_config
.
output_type
}
which does not use metric_list"
"metric list will be unused."
)
if
self
.
_config
.
output_type
==
"loglikelihood"
:
metric_list
=
[
"perplexity"
,
"acc"
]
elif
self
.
_config
.
output_type
==
"loglikelihood_rolling"
:
metric_list
=
[
"word_perplexity"
,
"byte_perplexity"
,
"bits_per_byte"
,
]
elif
self
.
_config
.
output_type
==
"multiple_choice"
:
metric_list
=
[
"acc"
,
"acc_norm"
]
for
metric_name
in
metric_list
:
self
.
_aggregation_list
[
metric_name
]
=
AGGREGATION_REGISTRY
[
"mean"
]
self
.
_higher_is_better
[
metric_name
]
=
HIGHER_IS_BETTER_REGISTRY
[
metric_name
]
self
.
download
(
self
.
_config
.
dataset_kwargs
)
self
.
download
(
self
.
_config
.
dataset_kwargs
)
self
.
_training_docs
=
None
self
.
_training_docs
=
None
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment