Skip to content

GitLab

Unverified Commit 6c340da4 authored by youkaichao's avatar youkaichao Committed by GitHub
Browse files

[misc] log info messages by default for hanging / busy / idle (#25627) 


Signed-off-by: default avataryoukaichao <youkaichao@gmail.com>
parent 2f171176
Show whitespace changes
Inline Side-by-side
Showing with 6 additions and 4 deletions
vllm/distributed/device_communicators/shm_broadcast.py
View file @ 6c340da4
...@@ -390,10 +390,11 @@ class MessageQueue: ...@@ -390,10 +390,11 @@ class MessageQueue:
# if we wait for a long time, log a message # if we wait for a long time, log a message
if (time.monotonic() - start_time if (time.monotonic() - start_time
> VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning): > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
logger.debug( logger.info(
("No available shared memory broadcast block found" ("No available shared memory broadcast block found"
" in %s seconds. This typically happens when some" " in %s seconds. This typically happens when some"
" processes are hanging."), " processes are hanging, doing some time-consuming"
" work (e.g. compilation), or sitting idle."),
VLLM_RINGBUFFER_WARNING_INTERVAL, VLLM_RINGBUFFER_WARNING_INTERVAL,
) )
n_warning += 1 n_warning += 1
...@@ -454,10 +455,11 @@ class MessageQueue: ...@@ -454,10 +455,11 @@ class MessageQueue:
# if we wait for a long time, log a message # if we wait for a long time, log a message
if (time.monotonic() - start_time if (time.monotonic() - start_time
> VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning): > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning):
logger.debug( logger.info(
("No available shared memory broadcast block found" ("No available shared memory broadcast block found"
" in %s seconds. This typically happens when some" " in %s seconds. This typically happens when some"
" processes are hanging."), " processes are hanging, doing some time-consuming"
" work (e.g. compilation), or sitting idle."),
VLLM_RINGBUFFER_WARNING_INTERVAL, VLLM_RINGBUFFER_WARNING_INTERVAL,
) )
n_warning += 1 n_warning += 1
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment