Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
b64e592d
Commit
b64e592d
authored
Nov 24, 2023
by
lintangsutawika
Browse files
fix missing lines
parent
47933b98
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
9 additions
and
7 deletions
+9
-7
lm_eval/__main__.py
lm_eval/__main__.py
+4
-2
lm_eval/evaluator.py
lm_eval/evaluator.py
+5
-5
No files found.
lm_eval/__main__.py
View file @
b64e592d
...
@@ -111,7 +111,9 @@ def parse_eval_args() -> argparse.Namespace:
...
@@ -111,7 +111,9 @@ def parse_eval_args() -> argparse.Namespace:
help
=
(
help
=
(
"String arguments for model generation on greedy_until tasks,"
"String arguments for model generation on greedy_until tasks,"
" e.g. `temperature=0,top_k=0,top_p=0`"
" e.g. `temperature=0,top_k=0,top_p=0`"
),
)
)
parser
.
add_argument
(
"--verbosity"
,
"--verbosity"
,
type
=
str
,
type
=
str
,
default
=
"INFO"
,
default
=
"INFO"
,
...
@@ -216,7 +218,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
...
@@ -216,7 +218,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
check_integrity
=
args
.
check_integrity
,
check_integrity
=
args
.
check_integrity
,
write_out
=
args
.
write_out
,
write_out
=
args
.
write_out
,
log_samples
=
args
.
log_samples
,
log_samples
=
args
.
log_samples
,
gen_kwargs
=
args
.
gen_kwargs
gen_kwargs
=
args
.
gen_kwargs
,
)
)
if
results
is
not
None
:
if
results
is
not
None
:
...
...
lm_eval/evaluator.py
View file @
b64e592d
...
@@ -21,7 +21,7 @@ from lm_eval.utils import (
...
@@ -21,7 +21,7 @@ from lm_eval.utils import (
create_iterator
,
create_iterator
,
get_git_commit_hash
,
get_git_commit_hash
,
simple_parse_args_string
,
simple_parse_args_string
,
eval_logger
eval_logger
,
)
)
...
@@ -41,7 +41,7 @@ def simple_evaluate(
...
@@ -41,7 +41,7 @@ def simple_evaluate(
decontamination_ngrams_path
=
None
,
decontamination_ngrams_path
=
None
,
write_out
:
bool
=
False
,
write_out
:
bool
=
False
,
log_samples
:
bool
=
True
,
log_samples
:
bool
=
True
,
gen_kwargs
:
str
=
None
gen_kwargs
:
str
=
None
,
):
):
"""Instantiate and evaluate a model on a list of tasks.
"""Instantiate and evaluate a model on a list of tasks.
...
@@ -127,8 +127,8 @@ def simple_evaluate(
...
@@ -127,8 +127,8 @@ def simple_evaluate(
continue
continue
config
=
task_obj
.
_config
config
=
task_obj
.
_config
if
config
[
'
output_type
'
]
==
'
greedy_until
'
and
gen_kwargs
is
not
None
:
if
config
[
"
output_type
"
]
==
"
greedy_until
"
and
gen_kwargs
is
not
None
:
config
[
'
generation_kwargs
'
].
update
(
gen_kwargs
)
config
[
"
generation_kwargs
"
].
update
(
gen_kwargs
)
if
num_fewshot
is
not
None
:
if
num_fewshot
is
not
None
:
if
config
[
"num_fewshot"
]
>
0
:
if
config
[
"num_fewshot"
]
>
0
:
...
@@ -167,7 +167,7 @@ def simple_evaluate(
...
@@ -167,7 +167,7 @@ def simple_evaluate(
"use_cache"
:
use_cache
,
"use_cache"
:
use_cache
,
"limit"
:
limit
,
"limit"
:
limit
,
"bootstrap_iters"
:
bootstrap_iters
,
"bootstrap_iters"
:
bootstrap_iters
,
"gen_kwargs"
:
gen_kwargs
"gen_kwargs"
:
gen_kwargs
,
}
}
results
[
"git_hash"
]
=
get_git_commit_hash
()
results
[
"git_hash"
]
=
get_git_commit_hash
()
return
results
return
results
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment