Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
3285f030
Commit
3285f030
authored
May 19, 2025
by
Baber
Browse files
refactor filter hf to use new output classes
parent
451e73f1
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
11 additions
and
4 deletions
+11
-4
lm_eval/api/filter.py
lm_eval/api/filter.py
+1
-1
lm_eval/api/types.py
lm_eval/api/types.py
+7
-0
lm_eval/models/huggingface.py
lm_eval/models/huggingface.py
+3
-3
No files found.
lm_eval/api/filter.py
View file @
3285f030
...
@@ -44,7 +44,7 @@ class FilterEnsemble:
...
@@ -44,7 +44,7 @@ class FilterEnsemble:
def
apply
(
self
,
instances
:
List
[
Instance
])
->
None
:
def
apply
(
self
,
instances
:
List
[
Instance
])
->
None
:
resps
,
docs
=
zip
(
*
((
inst
.
resps
,
inst
.
doc
)
for
inst
in
instances
))
resps
,
docs
=
zip
(
*
((
inst
.
resps
,
inst
.
doc
)
for
inst
in
instances
))
resps
,
docs
=
list
(
resps
),
list
(
docs
)
resps
,
docs
=
list
(
[
r
.
text
]
for
y
in
resps
for
r
in
y
),
list
(
docs
)
for
f
in
self
.
filters
:
for
f
in
self
.
filters
:
# apply filters in sequence
# apply filters in sequence
...
...
lm_eval/api/types.py
View file @
3285f030
...
@@ -12,6 +12,13 @@ class GenerateInput:
...
@@ -12,6 +12,13 @@ class GenerateInput:
gen_kwargs
:
dict
gen_kwargs
:
dict
multimodal_arg
:
Optional
[
dict
]
=
None
multimodal_arg
:
Optional
[
dict
]
=
None
def
__iter__
(
self
):
return
(
iter
((
self
.
prompt
,
self
.
gen_kwargs
))
if
not
self
.
multimodal_arg
else
iter
((
self
.
prompt
,
self
.
gen_kwargs
,
self
.
multimodal_arg
))
)
@
dataclass
@
dataclass
class
GenerateOutput
:
class
GenerateOutput
:
...
...
lm_eval/models/huggingface.py
View file @
3285f030
...
@@ -1321,8 +1321,8 @@ class HFLM(TemplateLM):
...
@@ -1321,8 +1321,8 @@ class HFLM(TemplateLM):
# padded context length. this is useful to simplify the batching logic and more importantly to make
# padded context length. this is useful to simplify the batching logic and more importantly to make
# automatic adaptive batches much much easier to implement
# automatic adaptive batches much much easier to implement
# - any OOMs will happen right away rather than near the end
# - any OOMs will happen right away rather than near the end
toks
=
self
.
tok_encode
(
req
[
0
]
)
toks
=
self
.
tok_encode
(
req
.
prompt
)
return
-
len
(
toks
),
req
[
0
]
return
-
len
(
toks
),
req
.
prompt
pbar
=
tqdm
(
pbar
=
tqdm
(
total
=
len
(
requests
),
total
=
len
(
requests
),
...
@@ -1358,7 +1358,7 @@ class HFLM(TemplateLM):
...
@@ -1358,7 +1358,7 @@ class HFLM(TemplateLM):
[
reg
.
args
for
reg
in
requests
],
[
reg
.
args
for
reg
in
requests
],
sort_fn
=
_collate
,
sort_fn
=
_collate
,
group_by
=
"gen_kwargs"
,
group_by
=
"gen_kwargs"
,
group_fn
=
lambda
x
:
x
[
1
]
,
group_fn
=
lambda
x
:
x
.
gen_kwargs
,
)
)
chunks
=
re_ords
.
get_batched
(
n
=
batch_size
,
batch_fn
=
batch_fn
)
chunks
=
re_ords
.
get_batched
(
n
=
batch_size
,
batch_fn
=
batch_fn
)
eos
=
self
.
tok_decode
(
self
.
eot_token_id
,
skip_special_tokens
=
False
)
eos
=
self
.
tok_decode
(
self
.
eot_token_id
,
skip_special_tokens
=
False
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment