Commit 75c249a2 authored by Hongkuan Zhou's avatar Hongkuan Zhou Committed by GitHub
Browse files

fix: wrong indent -> only one worker metric (#161)

parent 7713e25c
...@@ -66,20 +66,20 @@ class CustomRouter: ...@@ -66,20 +66,20 @@ class CustomRouter:
if metrics: if metrics:
for endpoint in metrics.endpoints: for endpoint in metrics.endpoints:
worker_id = endpoint.worker_id worker_id = endpoint.worker_id
worker_metrics[worker_id] = { worker_metrics[worker_id] = {
"gpu_cache_usage_perc": endpoint.gpu_cache_usage_perc "gpu_cache_usage_perc": endpoint.gpu_cache_usage_perc
if hasattr(endpoint, "gpu_cache_usage_perc") if hasattr(endpoint, "gpu_cache_usage_perc")
else 0.0, else 0.0,
"num_requests_waiting": endpoint.num_requests_waiting "num_requests_waiting": endpoint.num_requests_waiting
if hasattr(endpoint, "num_requests_waiting") if hasattr(endpoint, "num_requests_waiting")
else 0.0, else 0.0,
"gpu_prefix_cache_hit_rate": endpoint.gpu_prefix_cache_hit_rate "gpu_prefix_cache_hit_rate": endpoint.gpu_prefix_cache_hit_rate
if hasattr(endpoint, "gpu_prefix_cache_hit_rate") if hasattr(endpoint, "gpu_prefix_cache_hit_rate")
else 0.0, else 0.0,
} }
max_waiting = max( max_waiting = max(
max_waiting, worker_metrics[worker_id]["num_requests_waiting"] max_waiting, worker_metrics[worker_id]["num_requests_waiting"]
) )
# Get all worker IDs from the client. This is needed because scores / metrics may not have values for all workers # Get all worker IDs from the client. This is needed because scores / metrics may not have values for all workers
# and we want all workers to be considered in the logit calculation # and we want all workers to be considered in the logit calculation
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment