Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
9f698c20
Commit
9f698c20
authored
May 07, 2024
by
lintangsutawika
Browse files
sort set to False by default, fix predict_only arg
parent
8d59330b
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
7 additions
and
4 deletions
+7
-4
lm_eval/evaluator.py
lm_eval/evaluator.py
+2
-2
lm_eval/utils.py
lm_eval/utils.py
+5
-2
No files found.
lm_eval/evaluator.py
View file @
9f698c20
...
...
@@ -22,7 +22,7 @@ from lm_eval.evaluator_utils import (
print_writeout
,
run_task_tests
,
)
from
lm_eval.logging
_
utils
import
add_env_info
,
get_git_commit_hash
from
lm_eval.logging
.
utils
import
add_env_info
,
get_git_commit_hash
from
lm_eval.tasks
import
(
ConfigurableGroup
,
ConfigurableTask
,
...
...
@@ -224,7 +224,7 @@ def simple_evaluate(
if
isinstance
(
task_obj
,
dict
):
adjusted_task_dict
=
{
**
adjusted_task_dict
,
**
{
task_name
:
_adjust_config
(
task_obj
)},
**
{
task_name
:
_adjust_config
(
task_obj
,
predict_only
)},
}
else
:
...
...
lm_eval/utils.py
View file @
9f698c20
...
...
@@ -242,7 +242,7 @@ class Reorderer:
return
res
def
make_table
(
result_dict
,
column
:
str
=
"results"
,
sort_results
:
bool
=
Tru
e
):
def
make_table
(
result_dict
,
column
:
str
=
"results"
,
sort_results
:
bool
=
Fals
e
):
"""Generate table of results."""
from
pytablewriter
import
LatexTableWriter
,
MarkdownTableWriter
...
...
@@ -276,7 +276,10 @@ def make_table(result_dict, column: str = "results", sort_results: bool = True):
for
k
in
keys
:
dic
=
result_dict
[
column
][
k
]
version
=
result_dict
[
"versions"
].
get
(
k
,
" N/A"
)
n
=
str
(
result_dict
[
"n-shot"
][
k
])
if
k
in
result_dict
[
"n-shot"
]:
n
=
str
(
result_dict
[
"n-shot"
][
k
])
else
:
n
=
" "
if
"alias"
in
dic
:
k
=
dic
.
pop
(
"alias"
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment