"ssh:/git@developer.sourcefind.cn:2222/OpenDAS/vllm_cscc.git" did not exist on "0b544e6476eb88c3c011522c1feea6f16edf7022"
Unverified Commit acb54ca8 authored by Rabi Mishra's avatar Rabi Mishra Committed by GitHub
Browse files

Intialize io_thread_pool attribute in the beginning. (#18331)


Signed-off-by: default avatarrabi <ramishra@redhat.com>
parent 6e0fd34d
...@@ -50,6 +50,7 @@ class MultiprocExecutor(Executor): ...@@ -50,6 +50,7 @@ class MultiprocExecutor(Executor):
self.is_failed = False self.is_failed = False
self.shutdown_event = threading.Event() self.shutdown_event = threading.Event()
self.failure_callback: Optional[FailureCallback] = None self.failure_callback: Optional[FailureCallback] = None
self.io_thread_pool: Optional[ThreadPoolExecutor] = None
self.world_size = self.parallel_config.world_size self.world_size = self.parallel_config.world_size
tensor_parallel_size = self.parallel_config.tensor_parallel_size tensor_parallel_size = self.parallel_config.tensor_parallel_size
...@@ -107,7 +108,6 @@ class MultiprocExecutor(Executor): ...@@ -107,7 +108,6 @@ class MultiprocExecutor(Executor):
# For pipeline parallel, we use a thread pool for asynchronous # For pipeline parallel, we use a thread pool for asynchronous
# execute_model. # execute_model.
self.io_thread_pool: Optional[ThreadPoolExecutor] = None
if self.max_concurrent_batches > 1: if self.max_concurrent_batches > 1:
# Note: must use only 1 IO thread to keep dequeue sequence # Note: must use only 1 IO thread to keep dequeue sequence
# from the response queue # from the response queue
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment