Unverified Commit 21f4f1c9 authored by Harry Mellor's avatar Harry Mellor Committed by GitHub
Browse files

Improve static type checking in `LoRAModelRunnerMixin` (#17104)


Signed-off-by: default avatarHarry Mellor <19981378+hmellor@users.noreply.github.com>
parent b0c1f620
...@@ -28,20 +28,16 @@ class LoRAModelRunnerMixin: ...@@ -28,20 +28,16 @@ class LoRAModelRunnerMixin:
scheduler_config: SchedulerConfig, scheduler_config: SchedulerConfig,
lora_config: LoRAConfig, device: str) -> nn.Module: lora_config: LoRAConfig, device: str) -> nn.Module:
assert supports_lora( if not supports_lora(model):
model), f"{model.__class__.__name__} does not support LoRA yet." raise ValueError(
f"{model.__class__.__name__} does not support LoRA yet.")
if supports_multimodal(model): if supports_multimodal(model):
logger.warning("Regarding multimodal models, vLLM currently " logger.warning("Regarding multimodal models, vLLM currently "
"only supports adding LoRA to language model.") "only supports adding LoRA to language model.")
# It's necessary to distinguish between the max_position_embeddings # Use get_text_config() in case of multimodal models
# of VLMs and LLMs. text_config = model_config.hf_config.get_text_config()
if hasattr(model.config, "max_position_embeddings"):
max_pos_embeddings = model.config.max_position_embeddings
else:
max_pos_embeddings = (
model.config.text_config.max_position_embeddings)
# Add LoRA Manager to the Model Runner # Add LoRA Manager to the Model Runner
self.lora_manager = LRUCacheWorkerLoRAManager( self.lora_manager = LRUCacheWorkerLoRAManager(
...@@ -52,7 +48,7 @@ class LoRAModelRunnerMixin: ...@@ -52,7 +48,7 @@ class LoRAModelRunnerMixin:
device, device,
model.embedding_modules, model.embedding_modules,
model.embedding_padding_modules, model.embedding_padding_modules,
max_position_embeddings=max_pos_embeddings, max_position_embeddings=text_config.max_position_embeddings,
) )
return self.lora_manager.create_lora_manager(model) return self.lora_manager.create_lora_manager(model)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment