Unverified Commit 0d5bf022 authored by Graham King's avatar Graham King Committed by GitHub
Browse files

fix: Only read prompt embeds from request if globally enabled (#8248)


Signed-off-by: default avatarGraham King <grahamk@nvidia.com>
parent d644d88d
...@@ -1338,6 +1338,22 @@ class BaseWorkerHandler(ABC, Generic[RequestT, ResponseT]): ...@@ -1338,6 +1338,22 @@ class BaseWorkerHandler(ABC, Generic[RequestT, ResponseT]):
embedding_sequence_length = None embedding_sequence_length = None
if "prompt_embeds" in request and request["prompt_embeds"]: if "prompt_embeds" in request and request["prompt_embeds"]:
if not self.config.engine_args.enable_prompt_embeds:
msg = (
"Set `--enable-prompt-embeds` to allow `prompt_embeds` in request."
)
logger.error(
f"Rejected prompt_embeds for {log_prefix.lower().strip() or 'request'} "
f"{request_id}: {msg}"
)
return (
None,
None,
{
"finish_reason": f"error: Invalid prompt_embeds: {msg}",
"token_ids": [],
},
)
try: try:
( (
prompt, prompt,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment