Unverified Commit 98aee612 authored by Wentao Ye's avatar Wentao Ye Committed by GitHub
Browse files

[Log] Only Print Profiler Results on Rank 0 (#23370)


Signed-off-by: default avataryewentao256 <zhyanwentao@126.com>
parent 598bd74c
...@@ -400,6 +400,8 @@ class Worker(WorkerBase): ...@@ -400,6 +400,8 @@ class Worker(WorkerBase):
self.profiler.start() self.profiler.start()
else: else:
self.profiler.stop() self.profiler.stop()
# only print profiler results on rank 0
if self.local_rank == 0:
print(self.profiler.key_averages().table( print(self.profiler.key_averages().table(
sort_by="self_cuda_time_total")) sort_by="self_cuda_time_total"))
......
...@@ -128,8 +128,10 @@ class Worker(LocalOrDistributedWorkerBase): ...@@ -128,8 +128,10 @@ class Worker(LocalOrDistributedWorkerBase):
if self.profiler is None: if self.profiler is None:
raise RuntimeError("Profiler is not enabled.") raise RuntimeError("Profiler is not enabled.")
self.profiler.stop() self.profiler.stop()
print( # only print profiler results on rank 0
self.profiler.key_averages().table(sort_by="self_cuda_time_total")) if self.local_rank == 0:
print(self.profiler.key_averages().table(
sort_by="self_cuda_time_total"))
def sleep(self, level: int = 1) -> None: def sleep(self, level: int = 1) -> None:
free_bytes_before_sleep = torch.cuda.mem_get_info()[0] free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment