Unverified Commit d2d97f18 authored by Ameen Patel's avatar Ameen Patel Committed by GitHub
Browse files

fix: inherit tool_call_parser and reasoning_parser in LoRA model registration (#7559)


Signed-off-by: default avatarAmeenP <ameenp360@gmail.com>
parent de6e4f30
...@@ -40,6 +40,7 @@ from dynamo.common.utils.time_section import time_and_log_code_section ...@@ -40,6 +40,7 @@ from dynamo.common.utils.time_section import time_and_log_code_section
from dynamo.llm import ( from dynamo.llm import (
KvEventPublisher, KvEventPublisher,
ModelInput, ModelInput,
ModelRuntimeConfig,
ModelType, ModelType,
lora_name_to_id, lora_name_to_id,
register_model, register_model,
...@@ -776,6 +777,14 @@ class BaseWorkerHandler(ABC, Generic[RequestT, ResponseT]): ...@@ -776,6 +777,14 @@ class BaseWorkerHandler(ABC, Generic[RequestT, ResponseT]):
"lora_id": lora_id, "lora_id": lora_id,
} }
runtime_config = ModelRuntimeConfig()
runtime_config.tool_call_parser = (
self.config.dyn_tool_call_parser
)
runtime_config.reasoning_parser = (
self.config.dyn_reasoning_parser
)
# Publish with format: v1/mdc/dynamo/backend/generate/{instance_id}/{lora_slug} # Publish with format: v1/mdc/dynamo/backend/generate/{instance_id}/{lora_slug}
await register_model( await register_model(
model_input=ModelInput.Tokens, model_input=ModelInput.Tokens,
...@@ -783,6 +792,7 @@ class BaseWorkerHandler(ABC, Generic[RequestT, ResponseT]): ...@@ -783,6 +792,7 @@ class BaseWorkerHandler(ABC, Generic[RequestT, ResponseT]):
endpoint=self.generate_endpoint, endpoint=self.generate_endpoint,
model_path=self.config.model, model_path=self.config.model,
kv_cache_block_size=self.config.engine_args.block_size, kv_cache_block_size=self.config.engine_args.block_size,
runtime_config=runtime_config,
user_data=user_data, user_data=user_data,
lora_name=lora_name, lora_name=lora_name,
base_model_path=self.config.model, base_model_path=self.config.model,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment