Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
3a9bcc3f
Unverified
Commit
3a9bcc3f
authored
Aug 27, 2025
by
Baber Abbasi
Committed by
GitHub
Aug 27, 2025
Browse files
pacify pre-commit (#3268)
parent
a35eb973
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
20 additions
and
8 deletions
+20
-8
lm_eval/models/optimum_lm.py
lm_eval/models/optimum_lm.py
+3
-1
lm_eval/tasks/mmlu_prox/mmlu_prox_lite_config_generator.py
lm_eval/tasks/mmlu_prox/mmlu_prox_lite_config_generator.py
+1
-1
tests/models/test_openvino.py
tests/models/test_openvino.py
+16
-6
No files found.
lm_eval/models/optimum_lm.py
View file @
3a9bcc3f
...
...
@@ -76,7 +76,9 @@ class OptimumLM(HFLM):
"PIPELINE_PARALLEL"
)
model_cls
=
OVModelForCausalLM
if
self
.
backend
==
"causal"
else
OVModelForSeq2SeqLM
model_cls
=
(
OVModelForCausalLM
if
self
.
backend
==
"causal"
else
OVModelForSeq2SeqLM
)
self
.
_model
=
model_cls
.
from_pretrained
(
pretrained
,
revision
=
revision
,
...
...
lm_eval/tasks/mmlu_prox/mmlu_prox_lite_config_generator.py
View file @
3a9bcc3f
...
...
@@ -66,7 +66,7 @@ if __name__ == "__main__":
line
=
line
.
format
(
lang
=
lang_abbr
)
if
"{ans_regex}"
in
line
:
ans_regex
=
lang_lib_list
[
-
1
].
replace
(
"({})"
,
"\(?([ABCDEFGHIJ])\)?"
"({})"
,
r
"\(?([ABCDEFGHIJ])\)?"
)
if
lang_abbr
==
"en"
:
ans_regex
=
ans_regex
.
lstrip
(
"the"
).
strip
()
...
...
tests/models/test_openvino.py
View file @
3a9bcc3f
...
...
@@ -11,9 +11,21 @@ from lm_eval.api.registry import get_model
SUPPORTED_ARCHITECTURES_TASKS
=
[
(
"causal"
,
"facebook/opt-125m"
,
"lambada_openai"
,),
(
"causal"
,
"hf-internal-testing/tiny-random-gpt2"
,
"wikitext"
,),
(
"seq2seq"
,
"hf-internal-testing/tiny-random-t5"
,
"sst2"
,),
(
"causal"
,
"facebook/opt-125m"
,
"lambada_openai"
,
),
(
"causal"
,
"hf-internal-testing/tiny-random-gpt2"
,
"wikitext"
,
),
(
"seq2seq"
,
"hf-internal-testing/tiny-random-t5"
,
"sst2"
,
),
]
...
...
@@ -21,9 +33,7 @@ SUPPORTED_ARCHITECTURES_TASKS = [
def
test_evaluator
(
backend
,
model_id
,
task
):
with
tempfile
.
TemporaryDirectory
()
as
tmpdirname
:
model_cls
=
OVModelForCausalLM
if
backend
==
"causal"
else
OVModelForSeq2SeqLM
model
=
model_cls
.
from_pretrained
(
model_id
,
export
=
True
,
use_cache
=
True
)
model
=
model_cls
.
from_pretrained
(
model_id
,
export
=
True
,
use_cache
=
True
)
model
.
save_pretrained
(
tmpdirname
)
tokenizer
=
AutoTokenizer
.
from_pretrained
(
model_id
)
tokenizer
.
save_pretrained
(
tmpdirname
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment