Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
dynamo
Commits
75c249a2
Commit
75c249a2
authored
Mar 14, 2025
by
Hongkuan Zhou
Committed by
GitHub
Mar 14, 2025
Browse files
fix: wrong indent -> only one worker metric (#161)
parent
7713e25c
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
14 additions
and
14 deletions
+14
-14
examples/python_rs/llm/vllm/kv_router.py
examples/python_rs/llm/vllm/kv_router.py
+14
-14
No files found.
examples/python_rs/llm/vllm/kv_router.py
View file @
75c249a2
...
@@ -66,20 +66,20 @@ class CustomRouter:
...
@@ -66,20 +66,20 @@ class CustomRouter:
if
metrics
:
if
metrics
:
for
endpoint
in
metrics
.
endpoints
:
for
endpoint
in
metrics
.
endpoints
:
worker_id
=
endpoint
.
worker_id
worker_id
=
endpoint
.
worker_id
worker_metrics
[
worker_id
]
=
{
worker_metrics
[
worker_id
]
=
{
"gpu_cache_usage_perc"
:
endpoint
.
gpu_cache_usage_perc
"gpu_cache_usage_perc"
:
endpoint
.
gpu_cache_usage_perc
if
hasattr
(
endpoint
,
"gpu_cache_usage_perc"
)
if
hasattr
(
endpoint
,
"gpu_cache_usage_perc"
)
else
0.0
,
else
0.0
,
"num_requests_waiting"
:
endpoint
.
num_requests_waiting
"num_requests_waiting"
:
endpoint
.
num_requests_waiting
if
hasattr
(
endpoint
,
"num_requests_waiting"
)
if
hasattr
(
endpoint
,
"num_requests_waiting"
)
else
0.0
,
else
0.0
,
"gpu_prefix_cache_hit_rate"
:
endpoint
.
gpu_prefix_cache_hit_rate
"gpu_prefix_cache_hit_rate"
:
endpoint
.
gpu_prefix_cache_hit_rate
if
hasattr
(
endpoint
,
"gpu_prefix_cache_hit_rate"
)
if
hasattr
(
endpoint
,
"gpu_prefix_cache_hit_rate"
)
else
0.0
,
else
0.0
,
}
}
max_waiting
=
max
(
max_waiting
=
max
(
max_waiting
,
worker_metrics
[
worker_id
][
"num_requests_waiting"
]
max_waiting
,
worker_metrics
[
worker_id
][
"num_requests_waiting"
]
)
)
# Get all worker IDs from the client. This is needed because scores / metrics may not have values for all workers
# Get all worker IDs from the client. This is needed because scores / metrics may not have values for all workers
# and we want all workers to be considered in the logit calculation
# and we want all workers to be considered in the logit calculation
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment