Skip to content

GitLab

Unverified Commit f399182e authored by Chenheli Hua's avatar Chenheli Hua Committed by GitHub
Browse files

Run ruff format on a few files. (#24075) 


Signed-off-by: default avatarChenheli Hua <huachenheli@outlook.com>
parent 1c413105
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 947 additions and 690 deletions
tests/entrypoints/test_chat_utils.py
View file @ f399182e
This diff is collapsed.
vllm/entrypoints/chat_utils.py
View file @ f399182e
This diff is collapsed.
vllm/entrypoints/openai/serving_engine.py
View file @ f399182e
......@@ -82,16 +82,26 @@ from vllm.utils import (AsyncMicrobatchTokenizer, is_list_of,
logger = init_logger(__name__)
CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
EmbeddingCompletionRequest, RerankRequest,
ClassificationRequest, ScoreRequest,
TokenizeCompletionRequest]
CompletionLikeRequest = Union[
CompletionRequest,
DetokenizeRequest,
EmbeddingCompletionRequest,
RerankRequest,
ClassificationRequest,
ScoreRequest,
TokenizeCompletionRequest,
]
ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
TokenizeChatRequest]
SpeechToTextRequest = Union[TranscriptionRequest, TranslationRequest]
AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest, SpeechToTextRequest,
ResponsesRequest, IOProcessorRequest]
AnyRequest = Union[
CompletionLikeRequest,
ChatLikeRequest,
SpeechToTextRequest,
ResponsesRequest,
IOProcessorRequest,
]
AnyResponse = Union[
CompletionResponse,
......@@ -135,6 +145,7 @@ class RequestProcessingMixin(BaseModel):
Mixin for request processing,
handling prompt preparation and engine input.
"""
request_prompts: Optional[Sequence[RequestPrompt]] = []
engine_prompts: Optional[Union[list[EngineTokensPrompt],
list[EngineEmbedsPrompt]]] = []
......@@ -147,6 +158,7 @@ class ResponseGenerationMixin(BaseModel):
Mixin for response generation,
managing result generators and final batch results.
"""
result_generator: Optional[AsyncGenerator[tuple[int, Union[
RequestOutput, PoolingRequestOutput]], None]] = None
final_res_batch: list[Union[RequestOutput, PoolingRequestOutput]] = Field(
......@@ -155,8 +167,12 @@ class ResponseGenerationMixin(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True)
class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, BaseModel,
Generic[RequestT]):
class ServeContext(
RequestProcessingMixin,
ResponseGenerationMixin,
BaseModel,
Generic[RequestT],
):
# Shared across all requests
request: RequestT
raw_request: Optional[Request] = None
......@@ -298,8 +314,8 @@ class OpenAIServing:
truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens",
None)
if truncate_prompt_tokens is not None and \
truncate_prompt_tokens > self.max_model_len:
if (truncate_prompt_tokens is not None
and truncate_prompt_tokens > self.max_model_len):
return self.create_error_response(
"truncate_prompt_tokens value is "
"greater than max_model_len."
......@@ -344,10 +360,12 @@ class OpenAIServing:
return self.create_error_response(
"Request prompts not available")
self._log_inputs(request_id_item,
ctx.request_prompts[i],
params=pooling_params,
lora_request=ctx.lora_request)
self._log_inputs(
request_id_item,
ctx.request_prompts[i],
params=pooling_params,
lora_request=ctx.lora_request,
)
# Mypy has an existing bug related to inferring the variance of
# TypedDicts with `builtins.enumerate`:
......@@ -410,10 +428,11 @@ class OpenAIServing:
return self.create_error_response(str(e))
def create_error_response(
self,
message: str,
err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
self,
message: str,
err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
) -> ErrorResponse:
if self.log_error_stack:
exc_type, _, _ = sys.exc_info()
if exc_type is not None:
......@@ -424,10 +443,11 @@ class OpenAIServing:
message=message, type=err_type, code=status_code.value))
def create_streaming_error_response(
self,
message: str,
err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> str:
self,
message: str,
err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
) -> str:
json_str = json.dumps(
self.create_error_response(message=message,
err_type=err_type,
......@@ -438,25 +458,25 @@ class OpenAIServing:
self,
request: AnyRequest,
) -> Optional[ErrorResponse]:
error_response = None
if self._is_model_supported(request.model):
return None
if request.model in self.models.lora_requests:
return None
if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING and request.model and (
load_result := await self.models.resolve_lora(request.model)):
if (envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING and request.model and
(load_result := await self.models.resolve_lora(request.model))):
if isinstance(load_result, LoRARequest):
return None
if isinstance(load_result, ErrorResponse) and \
load_result.error.code == HTTPStatus.BAD_REQUEST.value:
if (isinstance(load_result, ErrorResponse) and
load_result.error.code == HTTPStatus.BAD_REQUEST.value):
error_response = load_result
return error_response or self.create_error_response(
message=f"The model `{request.model}` does not exist.",
err_type="NotFoundError",
status_code=HTTPStatus.NOT_FOUND)
status_code=HTTPStatus.NOT_FOUND,
)
def _get_active_default_mm_loras(
self, request: AnyRequest) -> Optional[LoRARequest]:
......@@ -487,7 +507,6 @@ class OpenAIServing:
request: AnyRequest,
supports_default_mm_loras: bool = False,
) -> Optional[LoRARequest]:
if request.model in self.models.lora_requests:
return self.models.lora_requests[request.model]
......@@ -548,13 +567,15 @@ class OpenAIServing:
prompt,
add_special_tokens=add_special_tokens,
truncation=True,
max_length=self.max_model_len)
max_length=self.max_model_len,
)
else:
encoded = await async_tokenizer(
prompt,
add_special_tokens=add_special_tokens,
truncation=True,
max_length=truncate_prompt_tokens)
max_length=truncate_prompt_tokens,
)
input_ids = encoded.input_ids
input_text = prompt
......@@ -595,16 +616,22 @@ class OpenAIServing:
# Note: EmbeddingRequest, ClassificationRequest,
# and ScoreRequest doesn't have max_tokens
if isinstance(request,
(EmbeddingChatRequest, EmbeddingCompletionRequest,
ScoreRequest, RerankRequest, ClassificationRequest)):
if isinstance(
request,
(
EmbeddingChatRequest,
EmbeddingCompletionRequest,
ScoreRequest,
RerankRequest,
ClassificationRequest,
),
):
# Note: input length can be up to the entire model context length
# since these requests don't generate tokens.
if token_num > self.max_model_len:
operations: dict[type[AnyRequest], str] = {
ScoreRequest: "score",
ClassificationRequest: "classification"
ClassificationRequest: "classification",
}
operation = operations.get(type(request),
"embedding generation")
......@@ -618,8 +645,11 @@ class OpenAIServing:
# Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens
# and does not require model context length validation
if isinstance(request, (TokenizeCompletionRequest, TokenizeChatRequest,
DetokenizeRequest)):
if isinstance(
request,
(TokenizeCompletionRequest, TokenizeChatRequest,
DetokenizeRequest),
):
return TextTokensPrompt(prompt=input_text,
prompt_token_ids=input_ids)
......@@ -639,8 +669,8 @@ class OpenAIServing:
f"{token_num} input tokens. Please reduce the length of "
"the input messages.")
if max_tokens is not None and \
token_num + max_tokens > self.max_model_len:
if (max_tokens is not None
and token_num + max_tokens > self.max_model_len):
raise ValueError(
"'max_tokens' or 'max_completion_tokens' is too large: "
f"{max_tokens}. This model's maximum context length is "
......@@ -745,13 +775,14 @@ class OpenAIServing:
tasks = []
for prompt_input in batch_inputs:
if prompt_input["is_tokens"] is False:
assert tokenizer is not None, \
"Tokenizer is required for text prompts"
assert tokenizer is not None, (
"Tokenizer is required for text prompts")
task = self._normalize_prompt_text_to_input(
request,
prompt_input["content"],
tokenizer=tokenizer,
add_special_tokens=add_special_tokens)
add_special_tokens=add_special_tokens,
)
else:
task = self._normalize_prompt_tokens_to_input(
request, prompt_input["content"], tokenizer=tokenizer)
......@@ -766,9 +797,14 @@ class OpenAIServing:
@overload
async def _preprocess_completion(
self,
request: Union[DetokenizeRequest, EmbeddingCompletionRequest,
RerankRequest, ClassificationRequest, ScoreRequest,
TokenizeCompletionRequest],
request: Union[
DetokenizeRequest,
EmbeddingCompletionRequest,
RerankRequest,
ClassificationRequest,
ScoreRequest,
TokenizeCompletionRequest,
],
tokenizer: Optional[AnyTokenizer],
input_or_inputs: Union[str, list[str], list[int], list[list[int]]],
add_special_tokens: bool = ...,
......@@ -783,8 +819,10 @@ class OpenAIServing:
input_or_inputs: Optional[Union[str, list[str], list[int],
list[list[int]]]],
add_special_tokens: bool = ...,
) -> tuple[list[Union[TextTokensPrompt, EmbedsPrompt]], list[Union[
EngineTokensPrompt, EngineEmbedsPrompt]]]:
) -> tuple[
list[Union[TextTokensPrompt, EmbedsPrompt]],
list[Union[EngineTokensPrompt, EngineEmbedsPrompt]],
]:
...
async def _preprocess_completion(
......@@ -794,32 +832,38 @@ class OpenAIServing:
input_or_inputs: Optional[Union[str, list[str], list[int],
list[list[int]]]],
add_special_tokens: bool = True,
) -> tuple[Union[list[TextTokensPrompt], list[Union[
TextTokensPrompt, EmbedsPrompt]]], Union[
list[EngineTokensPrompt], list[Union[EngineTokensPrompt,
EngineEmbedsPrompt]]]]:
if not isinstance(request,
CompletionRequest) and input_or_inputs is None:
) -> tuple[
Union[list[TextTokensPrompt], list[Union[TextTokensPrompt,
EmbedsPrompt]]],
Union[
list[EngineTokensPrompt],
list[Union[EngineTokensPrompt, EngineEmbedsPrompt]],
],
]:
if (not isinstance(request, CompletionRequest)
and input_or_inputs is None):
raise ValueError(
"Prompt embeds with non-completion requests is not"
" currently supported.")
(request_prompts_text, request_prompts_embeds
) = await self._tokenize_prompt_input_or_inputs_async(
request,
tokenizer,
input_or_inputs,
add_special_tokens=add_special_tokens,
)
(
request_prompts_text,
request_prompts_embeds,
) = await self._tokenize_prompt_input_or_inputs_async(
request,
tokenizer,
input_or_inputs,
add_special_tokens=add_special_tokens,
)
engine_prompts_text = [
EngineTokensPrompt(
prompt_token_ids=request_prompt_text["prompt_token_ids"])
for request_prompt_text in request_prompts_text
]
cache_salt = request.cache_salt if (
hasattr(request, "cache_salt")
and request.cache_salt is not None) else None
cache_salt = (request.cache_salt if
(hasattr(request, "cache_salt")
and request.cache_salt is not None) else None)
if cache_salt:
for prompt_text in engine_prompts_text:
prompt_text["cache_salt"] = cache_salt
......@@ -831,8 +875,8 @@ class OpenAIServing:
# non-completion requests and if we don't add the overload here,
# everywhere this function is used outside of serving_completion will
# need logic asserting that only text prompts are in the request.
if not isinstance(request,
CompletionRequest) and input_or_inputs is not None:
if (not isinstance(request, CompletionRequest)
and input_or_inputs is not None):
return request_prompts_text, engine_prompts_text
engine_prompts_embeds = [
......@@ -862,8 +906,11 @@ class OpenAIServing:
chat_template_kwargs: Optional[dict[str, Any]] = None,
tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,
add_special_tokens: bool = False,
) -> tuple[list[ConversationMessage], Sequence[RequestPrompt],
list[EngineTokensPrompt]]:
) -> tuple[
list[ConversationMessage],
Sequence[RequestPrompt],
list[EngineTokensPrompt],
]:
model_config = self.model_config
resolved_content_format = resolve_chat_template_content_format(
......@@ -925,8 +972,8 @@ class OpenAIServing:
if tokenizer is None:
assert isinstance(request_prompt, str), (
"Prompt has to be a string", \
"when the tokenizer is not initialised"
"Prompt has to be a string",
"when the tokenizer is not initialised",
)
prompt_inputs = TextTokensPrompt(prompt=request_prompt,
prompt_token_ids=[1])
......@@ -943,7 +990,8 @@ class OpenAIServing:
"Prompt has to be either a string or a list of token ids")
prompt_inputs = TextTokensPrompt(
prompt=tokenizer.decode(request_prompt),
prompt_token_ids=request_prompt)
prompt_token_ids=request_prompt,
)
engine_prompt = EngineTokensPrompt(
prompt_token_ids=prompt_inputs["prompt_token_ids"])
......@@ -1007,22 +1055,23 @@ class OpenAIServing:
prompt_token_ids=prompt_token_ids)
request_prompt = prompt_token_ids
# Update the sampling params.
sampling_params.max_tokens = (self.max_model_len -
len(prompt_token_ids))
sampling_params.max_tokens = self.max_model_len - len(
prompt_token_ids)
# OPTIMIZATION
priority = orig_priority - 1
@staticmethod
def _load_prompt_embeds(
prompt_embeds: Optional[Union[bytes, list[bytes]]],
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
) -> list[EmbedsPrompt]:
def _load_and_validate_embed(embed: bytes) -> EmbedsPrompt:
tensor = torch.load(io.BytesIO(
pybase64.b64decode(embed, validate=True)),
weights_only=True,
map_location=torch.device("cpu"))
tensor = torch.load(
io.BytesIO(pybase64.b64decode(embed, validate=True)),
weights_only=True,
map_location=torch.device("cpu"),
)
assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
torch.float32,
torch.bfloat16,
......@@ -1061,7 +1110,7 @@ class OpenAIServing:
prompt = inputs
elif isinstance(inputs, list):
prompt_token_ids = inputs
elif 'prompt_embeds' in inputs:
elif "prompt_embeds" in inputs:
prompt_embeds = inputs.get("prompt_embeds")
else:
prompt = inputs["prompt"]
......@@ -1101,10 +1150,12 @@ class OpenAIServing:
return raw_request.headers.get("X-Request-Id", default)
@staticmethod
def _get_decoded_token(logprob: Logprob,
token_id: int,
tokenizer: AnyTokenizer,
return_as_token_id: bool = False) -> str:
def _get_decoded_token(
logprob: Logprob,
token_id: int,
tokenizer: AnyTokenizer,
return_as_token_id: bool = False,
) -> str:
if return_as_token_id:
return f"token_id:{token_id}"
......@@ -1117,9 +1168,11 @@ class OpenAIServing:
return True
return self.models.is_base_model(model_name)
def _get_model_name(self,
model_name: Optional[str] = None,
lora_request: Optional[LoRARequest] = None) -> str:
def _get_model_name(
self,
model_name: Optional[str] = None,
lora_request: Optional[LoRARequest] = None,
) -> str:
if lora_request:
return lora_request.lora_name
if not model_name:
......@@ -1129,7 +1182,7 @@ class OpenAIServing:
def clamp_prompt_logprobs(
prompt_logprobs: Union[PromptLogprobs,
None]) -> Union[PromptLogprobs, None]:
None], ) -> Union[PromptLogprobs, None]:
if prompt_logprobs is None:
return prompt_logprobs
......@@ -1137,6 +1190,6 @@ def clamp_prompt_logprobs(
if logprob_dict is None:
continue
for logprob_values in logprob_dict.values():
if logprob_values.logprob == float('-inf'):
if logprob_values.logprob == float("-inf"):
logprob_values.logprob = -9999.0
return prompt_logprobs
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment