Unverified Commit a58f24e5 authored by zifeitong's avatar zifeitong Committed by GitHub
Browse files

[Bugfix] Fix torch.compile() error when using MultiprocessingGPUExecutor (#5229)

parent f42a006b
...@@ -34,6 +34,9 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor): ...@@ -34,6 +34,9 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
# Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id() os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
# Disable torch async compiling which won't work with daemonic processes
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
from torch.cuda import device_count from torch.cuda import device_count
assert world_size <= device_count(), ( assert world_size <= device_count(), (
"please set tensor_parallel_size to less than max local gpu count") "please set tensor_parallel_size to less than max local gpu count")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment