Skip to content

GitLab

Unverified Commit e444c13f authored by ishandhanani's avatar ishandhanani Committed by GitHub
Browse files

feat(engine): add bootstrap parameters to generate methods (dynamo) (#6075)

parent fee37d9e
Hide whitespace changes
Inline Side-by-side
Showing with 12 additions and 0 deletions
python/sglang/srt/entrypoints/engine.py
View file @ e444c13f
...@@ -163,6 +163,9 @@ class Engine(EngineBase): ...@@ -163,6 +163,9 @@ class Engine(EngineBase):
custom_logit_processor: Optional[Union[List[str], str]] = None, custom_logit_processor: Optional[Union[List[str], str]] = None,
return_hidden_states: bool = False, return_hidden_states: bool = False,
stream: bool = False, stream: bool = False,
bootstrap_host: Optional[Union[List[str], str]] = None,
bootstrap_port: Optional[Union[List[int], int]] = None,
bootstrap_room: Optional[Union[List[int], int]] = None,
) -> Union[Dict, Iterator[Dict]]: ) -> Union[Dict, Iterator[Dict]]:
""" """
The arguments of this function is the same as `sglang/srt/managers/io_struct.py::GenerateReqInput`. The arguments of this function is the same as `sglang/srt/managers/io_struct.py::GenerateReqInput`.
...@@ -181,6 +184,9 @@ class Engine(EngineBase): ...@@ -181,6 +184,9 @@ class Engine(EngineBase):
custom_logit_processor=custom_logit_processor, custom_logit_processor=custom_logit_processor,
return_hidden_states=return_hidden_states, return_hidden_states=return_hidden_states,
stream=stream, stream=stream,
bootstrap_host=bootstrap_host,
bootstrap_port=bootstrap_port,
bootstrap_room=bootstrap_room,
) )
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
generator = self.tokenizer_manager.generate_request(obj, None) generator = self.tokenizer_manager.generate_request(obj, None)
...@@ -227,6 +233,9 @@ class Engine(EngineBase): ...@@ -227,6 +233,9 @@ class Engine(EngineBase):
lora_path: Optional[List[Optional[str]]] = None, lora_path: Optional[List[Optional[str]]] = None,
custom_logit_processor: Optional[Union[List[str], str]] = None, custom_logit_processor: Optional[Union[List[str], str]] = None,
stream: bool = False, stream: bool = False,
bootstrap_host: Optional[Union[List[str], str]] = None,
bootstrap_port: Optional[Union[List[int], int]] = None,
bootstrap_room: Optional[Union[List[int], int]] = None,
) -> Union[Dict, AsyncIterator[Dict]]: ) -> Union[Dict, AsyncIterator[Dict]]:
""" """
The arguments of this function is the same as `sglang/srt/managers/io_struct.py::GenerateReqInput`. The arguments of this function is the same as `sglang/srt/managers/io_struct.py::GenerateReqInput`.
...@@ -244,6 +253,9 @@ class Engine(EngineBase): ...@@ -244,6 +253,9 @@ class Engine(EngineBase):
lora_path=lora_path, lora_path=lora_path,
stream=stream, stream=stream,
custom_logit_processor=custom_logit_processor, custom_logit_processor=custom_logit_processor,
bootstrap_host=bootstrap_host,
bootstrap_port=bootstrap_port,
bootstrap_room=bootstrap_room,
) )
generator = self.tokenizer_manager.generate_request(obj, None) generator = self.tokenizer_manager.generate_request(obj, None)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment