"docs/vscode:/vscode.git/clone" did not exist on "03fd76c57002747ded9bee5e7a5a699475cb4e9f"
Unverified Commit 08500011 authored by maobaolong's avatar maobaolong Committed by GitHub
Browse files

[Fix] Convert kv_transfer_config from dict to KVTransferConfig (#19262)

parent 861a0a0a
......@@ -10,6 +10,7 @@ from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
import cloudpickle
import torch.nn as nn
from pydantic import ValidationError
from tqdm.auto import tqdm
from typing_extensions import TypeVar, deprecated
......@@ -195,6 +196,23 @@ class LLM:
if isinstance(worker_cls, type):
kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)
if "kv_transfer_config" in kwargs and isinstance(
kwargs["kv_transfer_config"], dict):
from vllm.config import KVTransferConfig
raw_config_dict = kwargs["kv_transfer_config"]
try:
kwargs["kv_transfer_config"] = KVTransferConfig(
**raw_config_dict)
except ValidationError as e:
logger.error(
"Failed to convert 'kv_transfer_config' dict to "
"KVTransferConfig object. Dict: %s. Error: %s",
raw_config_dict, e)
# Consider re-raising a more specific vLLM error or ValueError
# to provide better context to the user.
raise ValueError(
f"Invalid 'kv_transfer_config' provided: {e}") from e
if hf_overrides is None:
hf_overrides = {}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment