Unverified Commit b82fc136 authored by aleksandaryanakiev's avatar aleksandaryanakiev Committed by GitHub
Browse files

[Anthropic][Frontend] Added chat_template_kwargs to /v1/messages (#40125)


Signed-off-by: default avatarAleksandar Yanakiev <alexander.yanakiev@discretestack.com>
Co-authored-by: default avatarAleksandar Yanakiev <alexander.yanakiev@discretestack.com>
parent e06de7f0
...@@ -117,6 +117,13 @@ class AnthropicMessagesRequest(BaseModel): ...@@ -117,6 +117,13 @@ class AnthropicMessagesRequest(BaseModel):
default=None, default=None,
description="KVTransfer parameters used for disaggregated serving.", description="KVTransfer parameters used for disaggregated serving.",
) )
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model") @field_validator("model")
@classmethod @classmethod
...@@ -212,6 +219,15 @@ class AnthropicCountTokensRequest(BaseModel): ...@@ -212,6 +219,15 @@ class AnthropicCountTokensRequest(BaseModel):
tool_choice: AnthropicToolChoice | None = None tool_choice: AnthropicToolChoice | None = None
tools: list[AnthropicTool] | None = None tools: list[AnthropicTool] | None = None
# vLLM-specific fields that are not in Anthropic spec
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model") @field_validator("model")
@classmethod @classmethod
def validate_model(cls, v): def validate_model(cls, v):
......
...@@ -323,6 +323,7 @@ class AnthropicServingMessages(OpenAIServingChat): ...@@ -323,6 +323,7 @@ class AnthropicServingMessages(OpenAIServingChat):
return ChatCompletionRequest( return ChatCompletionRequest(
model=anthropic_request.model, model=anthropic_request.model,
messages=openai_messages, messages=openai_messages,
chat_template_kwargs=anthropic_request.chat_template_kwargs,
) )
return ChatCompletionRequest( return ChatCompletionRequest(
...@@ -335,6 +336,7 @@ class AnthropicServingMessages(OpenAIServingChat): ...@@ -335,6 +336,7 @@ class AnthropicServingMessages(OpenAIServingChat):
top_p=anthropic_request.top_p, top_p=anthropic_request.top_p,
top_k=anthropic_request.top_k, top_k=anthropic_request.top_k,
kv_transfer_params=anthropic_request.kv_transfer_params, kv_transfer_params=anthropic_request.kv_transfer_params,
chat_template_kwargs=anthropic_request.chat_template_kwargs,
) )
@classmethod @classmethod
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment