Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
9d36354e
Unverified
Commit
9d36354e
authored
Nov 22, 2024
by
Baber Abbasi
Committed by
GitHub
Nov 23, 2024
Browse files
parse tokenizer_backend=None properly (#2509)
parent
867413f8
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
4 additions
and
2 deletions
+4
-2
lm_eval/models/api_models.py
lm_eval/models/api_models.py
+4
-2
No files found.
lm_eval/models/api_models.py
View file @
9d36354e
...
@@ -62,7 +62,7 @@ class TemplateAPI(TemplateLM):
...
@@ -62,7 +62,7 @@ class TemplateAPI(TemplateLM):
# however the requests can be sent as a string if the API doesn't support token inputs.
# however the requests can be sent as a string if the API doesn't support token inputs.
# use tokenized_requests=False
# use tokenized_requests=False
tokenizer_backend
:
Optional
[
tokenizer_backend
:
Optional
[
Literal
[
"tiktoken"
,
"huggingface"
,
None
]
Literal
[
"tiktoken"
,
"huggingface"
,
"
None
"
,
"none"
]
]
=
"huggingface"
,
]
=
"huggingface"
,
truncate
:
bool
=
False
,
truncate
:
bool
=
False
,
# number of concurrent requests. More useful if not batching
# number of concurrent requests. More useful if not batching
...
@@ -116,7 +116,9 @@ class TemplateAPI(TemplateLM):
...
@@ -116,7 +116,9 @@ class TemplateAPI(TemplateLM):
"Concurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1."
"Concurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1."
)
)
self
.
_concurrent
=
int
(
num_concurrent
)
self
.
_concurrent
=
int
(
num_concurrent
)
self
.
tokenizer_backend
=
tokenizer_backend
self
.
tokenizer_backend
=
(
None
if
tokenizer_backend
in
(
"None"
,
"none"
)
else
tokenizer_backend
)
self
.
add_bos_token
=
add_bos_token
self
.
add_bos_token
=
add_bos_token
self
.
custom_prefix_token_id
=
custom_prefix_token_id
self
.
custom_prefix_token_id
=
custom_prefix_token_id
self
.
tokenized_requests
=
tokenized_requests
self
.
tokenized_requests
=
tokenized_requests
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment