Unverified Commit d2d97f18 authored by Ameen Patel's avatar Ameen Patel Committed by GitHub
Browse files

fix: inherit tool_call_parser and reasoning_parser in LoRA model registration (#7559)


Signed-off-by: default avatarAmeenP <ameenp360@gmail.com>
parent de6e4f30
......@@ -40,6 +40,7 @@ from dynamo.common.utils.time_section import time_and_log_code_section
from dynamo.llm import (
KvEventPublisher,
ModelInput,
ModelRuntimeConfig,
ModelType,
lora_name_to_id,
register_model,
......@@ -776,6 +777,14 @@ class BaseWorkerHandler(ABC, Generic[RequestT, ResponseT]):
"lora_id": lora_id,
}
runtime_config = ModelRuntimeConfig()
runtime_config.tool_call_parser = (
self.config.dyn_tool_call_parser
)
runtime_config.reasoning_parser = (
self.config.dyn_reasoning_parser
)
# Publish with format: v1/mdc/dynamo/backend/generate/{instance_id}/{lora_slug}
await register_model(
model_input=ModelInput.Tokens,
......@@ -783,6 +792,7 @@ class BaseWorkerHandler(ABC, Generic[RequestT, ResponseT]):
endpoint=self.generate_endpoint,
model_path=self.config.model,
kv_cache_block_size=self.config.engine_args.block_size,
runtime_config=runtime_config,
user_data=user_data,
lora_name=lora_name,
base_model_path=self.config.model,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment