Unverified Commit 8d4a01cb authored by yuhsuan-t's avatar yuhsuan-t Committed by GitHub
Browse files

Log the timestamps of each prefill/decode iteration (#6094)


Co-authored-by: default avataryuhsuan-t <12108766+yuhsaun-t@users.noreply.github.com>
parent a3398d84
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
# ============================================================================== # ==============================================================================
"""A scheduler that manages a tensor parallel GPU worker.""" """A scheduler that manages a tensor parallel GPU worker."""
import datetime
import faulthandler import faulthandler
import logging import logging
import os import os
...@@ -1313,10 +1314,12 @@ class Scheduler( ...@@ -1313,10 +1314,12 @@ class Scheduler(
f += f"#unbootstrapped-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, " f += f"#unbootstrapped-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, "
f += f"#queue-req: {len(self.waiting_queue)}, " f += f"#queue-req: {len(self.waiting_queue)}, "
f += f"#transferring-req: {len(self.disagg_prefill_inflight_queue)}, " f += f"#transferring-req: {len(self.disagg_prefill_inflight_queue)}, "
f += f"input throughput (token/s): {self.last_input_throughput:.2f} " f += f"input throughput (token/s): {self.last_input_throughput:.2f}, "
else: else:
f += f"#running-req: {running_bs}, " f += f"#running-req: {running_bs}, "
f += f"#queue-req: {len(self.waiting_queue)}" f += f"#queue-req: {len(self.waiting_queue)}, "
f += f"timestamp: {datetime.datetime.now().isoformat()}"
logger.info(f) logger.info(f)
...@@ -1378,7 +1381,8 @@ class Scheduler( ...@@ -1378,7 +1381,8 @@ class Scheduler(
msg += ( msg += (
f"cuda graph: {can_run_cuda_graph}, " f"cuda graph: {can_run_cuda_graph}, "
f"gen throughput (token/s): {self.last_gen_throughput:.2f}, " f"gen throughput (token/s): {self.last_gen_throughput:.2f}, "
f"#queue-req: {len(self.waiting_queue)}" f"#queue-req: {len(self.waiting_queue)}, "
f"timestamp: {datetime.datetime.now().isoformat()}"
) )
logger.info(msg) logger.info(msg)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment