START TIME: Thu Nov  9 12:04:37 CST 2023
e07r1n18
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
LRANK===============================2
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
LRANK===============================2
LRANK===============================2
LRANK===============================2
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
LRANK===============================2
LRANK===============================2
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================2
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================2
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================2
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
[2023-11-09 12:05:39,800] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,800] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,800] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,800] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,794] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,784] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,785] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,801] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,794] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,804] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,801] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,804] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,783] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,800] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,798] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,801] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,801] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,801] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,789] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,788] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,786] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,794] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,804] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,801] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,804] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,801] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,802] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,801] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,790] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,786] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,802] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,802] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,790] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,785] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,788] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,789] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,797] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,786] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,804] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,789] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,808] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,786] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,786] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,787] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,803] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,808] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,786] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,791] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,807] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,789] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,787] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,807] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,809] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,804] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,809] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,809] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,808] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,787] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,806] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,787] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,806] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,791] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,790] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,788] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,804] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,787] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,802] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,806] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,788] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,802] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,802] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,788] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,792] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,806] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,788] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,805] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:39,788] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-11-09 12:05:40,087] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=72, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,081] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=88, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=76, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,079] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=52, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=56, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,075] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=24, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=36, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=84, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,097] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=4, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=8, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=60, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=68, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=40, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,090] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=28, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=44, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,097] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=92, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,094] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=20, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,075] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=12, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,097] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=48, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=80, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,077] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=64, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,094] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=32, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=0, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,077] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=16, local_rank=0, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=1, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,094] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=57, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,082] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=89, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,077] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=25, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,094] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=33, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,094] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=85, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:622:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
[2023-11-09 12:05:40,077] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=65, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=41, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,094] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=69, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=21, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=37, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,098] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=93, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=81, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,094] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=61, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,088] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=73, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,077] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=17, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,097] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=49, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,094] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=9, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,099] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=5, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,077] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=13, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=77, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,081] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=53, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,080] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=45, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,092] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=29, local_rank=1, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,079] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=2, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,079] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=3, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,100] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=94, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=26, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=62, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,100] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=6, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,100] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=7, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,097] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=22, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=58, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=18, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,081] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=46, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,082] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=54, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=38, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=30, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,083] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=90, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,090] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=74, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=86, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=34, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,099] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=50, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=14, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,097] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=82, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=10, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=70, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,097] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=23, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=59, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=19, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,081] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=47, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,082] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=55, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=39, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,093] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=31, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,083] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=91, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,090] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=75, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=87, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=35, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,099] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=51, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=15, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,097] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=83, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=11, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,095] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=71, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,079] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=66, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=78, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,078] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=27, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=43, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,100] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=95, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=63, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,079] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=67, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=79, local_rank=3, world_size=96, master_addr=10.5.7.19, master_port=29500
[2023-11-09 12:05:40,096] [INFO] [comm.py:659:mpi_discovery] Discovered MPI settings of world_rank=42, local_rank=2, world_size=96, master_addr=10.5.7.19, master_port=29500
e07r1n18:15185:15185 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.19<0>
e07r1n18:15185:15185 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r1n18:15185:15185 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
RCCL version 2.13.4+hip5.4 HEAD:82f11f7
e07r1n18:15185:16349 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.19<0>
e07r1n18:15185:16349 [0] NCCL INFO Using network IB
e07r1n18:15189:15189 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.19<0>
e07r1n18:15187:15187 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.19<0>
e07r1n18:15188:15188 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.19<0>
e07r1n18:15189:15189 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r1n18:15187:15187 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r1n18:15188:15188 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r1n18:15189:15189 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r1n18:15187:15187 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r1n18:15188:15188 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n04:1826:1826 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.25<0>
e07r2n04:1826:1826 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n10:29369:29369 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.31<0>
e07r2n10:29369:29369 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n01:626:626 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.22<0>
e07r2n01:626:626 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n08:30380:30380 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.29<0>
e07r2n08:30380:30380 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r3n01:6046:6046 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.42<0>
e07r3n01:6046:6046 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n12:16035:16035 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.33<0>
e07r2n12:16035:16035 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n00:23099:23099 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.21<0>
e07r2n00:23099:23099 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n10:29372:29372 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.31<0>
e07r2n10:29372:29372 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n04:1827:1827 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.25<0>
e07r2n04:1827:1827 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n14:22275:22275 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.35<0>
e07r2n14:22275:22275 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r3n01:6048:6048 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.42<0>
e07r3n01:6048:6048 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n05:906:906 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.26<0>
e07r2n05:906:906 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n03:21863:21863 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.24<0>
e07r2n03:21863:21863 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r3n00:24490:24490 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.41<0>
e07r3n00:24490:24490 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r1n19:30602:30602 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.20<0>
e07r1n19:30602:30602 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n01:629:629 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.22<0>
e07r2n01:629:629 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r3n00:24491:24491 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.41<0>
e07r3n00:24491:24491 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r1n19:30600:30600 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.20<0>
e07r1n19:30600:30600 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n12:16038:16038 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.33<0>
e07r2n12:16037:16037 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.33<0>
e07r2n12:16038:16038 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n12:16037:16037 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n16:26995:26995 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.37<0>
e07r2n16:26995:26995 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n06:26608:26608 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.27<0>
e07r2n06:26608:26608 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n03:21864:21864 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.24<0>
e07r2n03:21864:21864 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n17:31184:31184 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.38<0>
e07r2n17:31184:31184 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n15:28148:28148 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.36<0>
e07r2n15:28148:28148 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n05:907:907 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.26<0>
e07r2n05:907:907 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r3n00:24493:24493 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.41<0>
e07r3n00:24493:24493 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n12:16039:16039 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.33<0>
e07r2n12:16039:16039 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n11:16526:16526 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.32<0>
e07r2n11:16526:16526 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n08:30377:30377 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.29<0>
e07r2n08:30377:30377 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n10:29371:29371 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.31<0>
e07r2n10:29371:29371 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n14:22276:22276 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.35<0>
e07r2n14:22276:22276 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n06:26609:26609 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.27<0>
e07r2n16:26996:26996 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.37<0>
e07r2n06:26609:26609 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n16:26996:26996 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n13:31206:31206 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.34<0>
e07r2n13:31206:31206 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n13:31207:31207 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.34<0>
e07r2n13:31207:31207 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r3n00:24492:24492 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.41<0>
e07r2n09:23023:23023 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.30<0>
e07r2n09:23023:23023 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r3n00:24492:24492 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n08:30378:30378 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.29<0>
e07r2n08:30378:30378 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n17:31185:31185 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.38<0>
e07r2n05:909:909 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.26<0>
e07r2n05:909:909 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n17:31185:31185 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n14:22278:22278 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.35<0>
e07r2n14:22278:22278 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n10:29373:29373 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.31<0>
e07r2n16:26997:26997 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.37<0>
e07r2n10:29373:29373 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n16:26997:26997 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n06:26611:26611 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.27<0>
e07r2n06:26611:26611 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n19:2107:2107 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.40<0>
e07r2n19:2107:2107 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n03:21866:21866 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.24<0>
e07r2n03:21866:21866 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r1n19:30599:30599 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.20<0>
e07r2n15:28145:28145 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.36<0>
e07r1n19:30599:30599 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n15:28150:28150 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.36<0>
e07r2n15:28145:28145 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n15:28150:28150 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n01:627:627 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.22<0>
e07r2n01:627:627 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n17:31186:31186 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.38<0>
e07r2n17:31186:31186 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n05:908:908 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.26<0>
e07r2n17:31187:31187 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.38<0>
e07r2n09:23024:23024 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.30<0>
e07r2n05:908:908 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n17:31187:31187 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n09:23024:23024 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n07:11608:11608 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.28<0>
e07r2n07:11608:11608 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n11:16527:16527 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.32<0>
e07r2n11:16527:16527 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n19:2109:2109 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.40<0>
e07r2n19:2109:2109 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n01:628:628 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.22<0>
e07r2n16:26998:26998 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.37<0>
e07r2n01:628:628 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n16:26998:26998 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n15:28147:28147 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.36<0>
e07r3n01:6047:6047 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.42<0>
e07r3n01:6047:6047 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n15:28147:28147 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n03:21865:21865 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.24<0>
e07r2n03:21865:21865 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n14:22277:22277 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.35<0>
e07r2n14:22277:22277 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n04:1829:1829 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.25<0>
e07r2n04:1829:1829 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n13:31209:31209 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.34<0>
e07r2n06:26610:26610 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.27<0>
e07r2n13:31209:31209 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n06:26610:26610 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r3n01:6049:6049 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.42<0>
e07r3n01:6049:6049 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n07:11607:11607 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.28<0>
e07r2n09:23026:23026 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.30<0>
e07r2n07:11607:11607 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n11:16529:16529 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.32<0>
e07r2n09:23026:23026 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n11:16529:16529 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n04:1828:1828 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.25<0>
e07r2n04:1828:1828 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n19:2110:2110 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.40<0>
e07r2n19:2110:2110 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n19:2111:2111 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.40<0>
e07r2n19:2111:2111 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n13:31210:31210 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.34<0>
e07r2n13:31210:31210 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r1n19:30601:30601 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.20<0>
e07r1n19:30601:30601 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n09:23025:23025 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.30<0>
e07r2n02:4756:4756 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.23<0>
e07r2n02:4756:4756 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n09:23025:23025 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n11:16528:16528 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.32<0>
e07r2n11:16528:16528 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n18:25918:25918 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.39<0>
e07r2n18:25918:25918 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n00:23103:23103 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.21<0>
e07r2n00:23103:23103 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n07:11610:11610 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.28<0>
e07r2n07:11610:11610 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n02:4758:4758 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.23<0>
e07r2n02:4758:4758 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n02:4760:4760 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.23<0>
e07r2n02:4759:4759 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.23<0>
e07r2n18:25919:25919 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.39<0>
e07r2n02:4760:4760 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n02:4759:4759 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n18:25919:25919 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n07:11609:11609 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.28<0>
e07r2n07:11609:11609 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n18:25917:25917 [0] NCCL INFO Bootstrap : Using ib0:11.5.7.39<0>
e07r2n18:25917:25917 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n18:25920:25920 [3] NCCL INFO Bootstrap : Using ib0:11.5.7.39<0>
e07r2n18:25920:25920 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n01:628:628 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n01:627:627 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n01:626:626 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n01:629:629 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r3n01:6049:6049 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r3n01:6046:6046 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r3n01:6048:6048 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r3n01:6047:6047 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r3n00:24490:24490 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r3n00:24493:24493 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r3n00:24491:24491 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r3n00:24492:24492 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n08:30379:30379 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.29<0>
e07r2n08:30379:30379 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n14:22276:22276 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n14:22275:22275 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n14:22278:22278 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n14:22277:22277 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n05:908:908 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n12:16038:16038 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n12:16039:16039 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n05:909:909 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n12:16035:16035 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n05:906:906 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n12:16037:16037 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n05:907:907 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n00:23099:23099 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n00:23103:23103 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n06:26610:26610 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n06:26608:26608 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n06:26609:26609 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n06:26611:26611 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n08:30377:30377 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n08:30379:30379 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n08:30378:30378 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n08:30380:30380 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n16:26998:26998 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n16:26995:26995 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n16:26996:26996 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n16:26997:26997 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n17:31187:31187 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n17:31184:31184 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n17:31186:31186 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n17:31185:31185 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n00:23101:23101 [1] NCCL INFO Bootstrap : Using ib0:11.5.7.21<0>
e07r2n00:23101:23101 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n11:16526:16526 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n11:16527:16527 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n11:16529:16529 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n11:16528:16528 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n00:23101:23101 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r1n19:30602:30602 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r1n19:30599:30599 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r1n19:30601:30601 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r1n19:30600:30600 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n07:11608:11608 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n07:11607:11607 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n07:11610:11610 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n07:11609:11609 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n19:2107:2107 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n19:2110:2110 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n19:2111:2111 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n19:2109:2109 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n02:4758:4758 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n02:4759:4759 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n02:4760:4760 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n02:4756:4756 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n13:31209:31209 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n00:23102:23102 [2] NCCL INFO Bootstrap : Using ib0:11.5.7.21<0>
e07r2n13:31210:31210 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n00:23102:23102 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e07r2n13:31206:31206 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n13:31207:31207 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n00:23102:23102 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n03:21865:21865 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n03:21864:21864 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n03:21866:21866 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n03:21863:21863 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n10:29369:29369 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n18:25917:25917 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n10:29371:29371 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n18:25920:25920 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n10:29373:29373 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n18:25919:25919 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n10:29372:29372 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n18:25918:25918 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n04:1829:1829 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n04:1827:1827 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n04:1828:1828 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n04:1826:1826 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n15:28150:28150 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n15:28147:28147 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n15:28148:28148 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n15:28145:28145 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n09:23025:23025 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n09:23026:23026 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n09:23024:23024 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r2n09:23023:23023 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e07r1n18:15189:16547 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.19<0>
e07r1n18:15189:16547 [3] NCCL INFO Using network IB
e07r1n18:15187:16550 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.19<0>
e07r1n18:15187:16550 [1] NCCL INFO Using network IB
e07r1n18:15188:16551 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.19<0>
e07r1n18:15188:16551 [2] NCCL INFO Using network IB
e07r2n01:628:1825 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.22<0>
e07r2n01:628:1825 [2] NCCL INFO Using network IB
e07r2n01:629:1819 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.22<0>
e07r2n01:629:1819 [3] NCCL INFO Using network IB
e07r2n01:627:1821 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.22<0>
e07r2n01:627:1821 [1] NCCL INFO Using network IB
e07r2n01:626:1823 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.22<0>
e07r2n01:626:1823 [0] NCCL INFO Using network IB
e07r3n01:6046:7322 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.42<0>
e07r3n01:6046:7322 [0] NCCL INFO Using network IB
e07r3n01:6047:7323 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.42<0>
e07r3n01:6047:7323 [1] NCCL INFO Using network IB
e07r3n01:6048:7326 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.42<0>
e07r3n01:6048:7326 [2] NCCL INFO Using network IB
e07r3n01:6049:7320 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.42<0>
e07r3n01:6049:7320 [3] NCCL INFO Using network IB
e07r3n00:24491:25708 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.41<0>
e07r3n00:24491:25708 [1] NCCL INFO Using network IB
e07r3n00:24490:25707 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.41<0>
e07r3n00:24490:25707 [0] NCCL INFO Using network IB
e07r3n00:24493:25710 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.41<0>
e07r3n00:24493:25710 [3] NCCL INFO Using network IB
e07r3n00:24492:25711 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.41<0>
e07r3n00:24492:25711 [2] NCCL INFO Using network IB
e07r2n14:22275:23497 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.35<0>
e07r2n14:22275:23497 [0] NCCL INFO Using network IB
e07r2n14:22276:23489 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.35<0>
e07r2n14:22276:23489 [1] NCCL INFO Using network IB
e07r2n14:22278:23496 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.35<0>
e07r2n14:22278:23496 [3] NCCL INFO Using network IB
e07r2n14:22277:23498 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.35<0>
e07r2n14:22277:23498 [2] NCCL INFO Using network IB
e07r2n12:16037:17013 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.33<0>
e07r2n12:16037:17013 [1] NCCL INFO Using network IB
e07r2n12:16039:17014 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.33<0>
e07r2n12:16039:17014 [3] NCCL INFO Using network IB
e07r2n12:16035:17011 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.33<0>
e07r2n12:16035:17011 [0] NCCL INFO Using network IB
e07r2n12:16038:17016 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.33<0>
e07r2n12:16038:17016 [2] NCCL INFO Using network IB
e07r2n00:23099:24278 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.21<0>
e07r2n00:23099:24278 [0] NCCL INFO Using network IB
e07r2n00:23103:24289 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.21<0>
e07r2n00:23103:24289 [3] NCCL INFO Using network IB
e07r2n08:30378:31693 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.29<0>
e07r2n08:30378:31693 [1] NCCL INFO Using network IB
e07r2n08:30377:31697 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.29<0>
e07r2n08:30377:31697 [0] NCCL INFO Using network IB
e07r2n08:30380:31690 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.29<0>
e07r2n08:30380:31690 [3] NCCL INFO Using network IB
e07r2n06:26608:27568 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.27<0>
e07r2n06:26608:27568 [0] NCCL INFO Using network IB
e07r2n06:26609:27563 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.27<0>
e07r2n06:26609:27563 [1] NCCL INFO Using network IB
e07r2n06:26611:27566 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.27<0>
e07r2n06:26611:27566 [3] NCCL INFO Using network IB
e07r2n06:26610:27569 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.27<0>
e07r2n06:26610:27569 [2] NCCL INFO Using network IB
e07r2n05:907:2100 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.26<0>
e07r2n05:907:2100 [1] NCCL INFO Using network IB
e07r2n05:909:2102 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.26<0>
e07r2n05:909:2102 [3] NCCL INFO Using network IB
e07r2n05:906:2099 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.26<0>
e07r2n05:906:2099 [0] NCCL INFO Using network IB
e07r2n05:908:2101 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.26<0>
e07r2n05:908:2101 [2] NCCL INFO Using network IB
e07r1n19:30600:31684 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.20<0>
e07r1n19:30600:31684 [1] NCCL INFO Using network IB
e07r2n11:16527:17685 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.32<0>
e07r2n11:16527:17685 [1] NCCL INFO Using network IB
e07r2n11:16526:17682 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.32<0>
e07r2n11:16526:17682 [0] NCCL INFO Using network IB
e07r1n19:30602:31679 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.20<0>
e07r1n19:30602:31679 [3] NCCL INFO Using network IB
e07r2n11:16529:17687 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.32<0>
e07r2n11:16529:17687 [3] NCCL INFO Using network IB
e07r2n17:31187:32200 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.38<0>
e07r2n17:31187:32200 [3] NCCL INFO Using network IB
e07r1n19:30599:31677 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.20<0>
e07r1n19:30599:31677 [0] NCCL INFO Using network IB
e07r1n19:30601:31683 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.20<0>
e07r1n19:30601:31683 [2] NCCL INFO Using network IB
e07r2n11:16528:17686 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.32<0>
e07r2n11:16528:17686 [2] NCCL INFO Using network IB
e07r2n16:26996:28158 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.37<0>
e07r2n16:26996:28158 [1] NCCL INFO Using network IB
e07r2n16:26995:28159 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.37<0>
e07r2n16:26995:28159 [0] NCCL INFO Using network IB
e07r2n16:26998:28161 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.37<0>
e07r2n16:26998:28161 [3] NCCL INFO Using network IB
e07r2n16:26997:28160 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.37<0>
e07r2n16:26997:28160 [2] NCCL INFO Using network IB
e07r2n17:31186:32202 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.38<0>
e07r2n17:31186:32202 [2] NCCL INFO Using network IB
e07r2n17:31184:32198 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.38<0>
e07r2n17:31184:32198 [0] NCCL INFO Using network IB
e07r2n17:31185:32201 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.38<0>
e07r2n17:31185:32201 [1] NCCL INFO Using network IB
e07r2n02:4760:5858 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.23<0>
e07r2n02:4760:5858 [3] NCCL INFO Using network IB
e07r2n02:4758:5857 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.23<0>
e07r2n02:4758:5857 [1] NCCL INFO Using network IB
e07r2n02:4759:5859 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.23<0>
e07r2n02:4759:5859 [2] NCCL INFO Using network IB
e07r2n02:4756:5860 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.23<0>
e07r2n02:4756:5860 [0] NCCL INFO Using network IB
e07r2n13:31209:32208 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.34<0>
e07r2n13:31209:32208 [2] NCCL INFO Using network IB
e07r2n13:31206:32207 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.34<0>
e07r2n13:31206:32207 [0] NCCL INFO Using network IB
e07r2n13:31210:32209 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.34<0>
e07r2n13:31210:32209 [3] NCCL INFO Using network IB
e07r2n13:31207:32206 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.34<0>
e07r2n13:31207:32206 [1] NCCL INFO Using network IB
e07r2n10:29369:30329 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.31<0>
e07r2n10:29369:30329 [0] NCCL INFO Using network IB
e07r2n10:29373:30327 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.31<0>
e07r2n10:29373:30327 [3] NCCL INFO Using network IB
e07r2n19:2109:3517 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.40<0>
e07r2n19:2109:3517 [1] NCCL INFO Using network IB
e07r2n10:29372:30328 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.31<0>
e07r2n10:29372:30328 [2] NCCL INFO Using network IB
e07r2n10:29371:30326 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.31<0>
e07r2n10:29371:30326 [1] NCCL INFO Using network IB
e07r2n19:2110:3521 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.40<0>
e07r2n19:2110:3521 [2] NCCL INFO Using network IB
e07r2n19:2107:3519 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.40<0>
e07r2n19:2107:3519 [0] NCCL INFO Using network IB
e07r2n19:2111:3520 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.40<0>
e07r2n19:2111:3520 [3] NCCL INFO Using network IB
e07r2n03:21863:23045 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.24<0>
e07r2n03:21863:23045 [0] NCCL INFO Using network IB
e07r2n03:21866:23047 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.24<0>
e07r2n03:21866:23047 [3] NCCL INFO Using network IB
e07r2n03:21864:23049 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.24<0>
e07r2n03:21864:23049 [1] NCCL INFO Using network IB
e07r2n03:21865:23050 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.24<0>
e07r2n03:21865:23050 [2] NCCL INFO Using network IB
e07r2n07:11610:12943 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.28<0>
e07r2n07:11610:12943 [3] NCCL INFO Using network IB
e07r2n18:25919:26914 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.39<0>
e07r2n18:25919:26914 [2] NCCL INFO Using network IB
e07r2n07:11608:12938 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.28<0>
e07r2n07:11608:12938 [1] NCCL INFO Using network IB
e07r2n07:11609:12944 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.28<0>
e07r2n07:11609:12944 [2] NCCL INFO Using network IB
e07r2n07:11607:12941 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.28<0>
e07r2n07:11607:12941 [0] NCCL INFO Using network IB
e07r2n18:25920:26909 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.39<0>
e07r2n18:25920:26909 [3] NCCL INFO Using network IB
e07r2n18:25917:26912 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.39<0>
e07r2n18:25917:26912 [0] NCCL INFO Using network IB
e07r2n18:25918:26911 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.39<0>
e07r2n18:25918:26911 [1] NCCL INFO Using network IB
e07r2n15:28150:29329 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.36<0>
e07r2n15:28150:29329 [3] NCCL INFO Using network IB
e07r2n15:28147:29326 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.36<0>
e07r2n15:28147:29326 [1] NCCL INFO Using network IB
e07r2n15:28148:29328 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.36<0>
e07r2n15:28148:29328 [2] NCCL INFO Using network IB
e07r2n15:28145:29327 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.36<0>
e07r2n15:28145:29327 [0] NCCL INFO Using network IB
e07r2n09:23023:24187 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.30<0>
e07r2n09:23023:24187 [0] NCCL INFO Using network IB
e07r2n09:23024:24184 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.30<0>
e07r2n09:23024:24184 [1] NCCL INFO Using network IB
e07r2n09:23026:24186 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.30<0>
e07r2n09:23026:24186 [3] NCCL INFO Using network IB
e07r2n09:23025:24185 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.30<0>
e07r2n09:23025:24185 [2] NCCL INFO Using network IB
e07r2n04:1827:3073 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.25<0>
e07r2n04:1827:3073 [1] NCCL INFO Using network IB
e07r2n04:1829:3075 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.25<0>
e07r2n04:1829:3075 [3] NCCL INFO Using network IB
e07r2n04:1828:3077 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.25<0>
e07r2n04:1828:3077 [2] NCCL INFO Using network IB
e07r2n04:1826:3076 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.25<0>
e07r2n04:1826:3076 [0] NCCL INFO Using network IB
e07r2n08:30379:31736 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.29<0>
e07r2n08:30379:31736 [2] NCCL INFO Using network IB
e07r2n00:23101:24353 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.21<0>
e07r2n00:23101:24353 [1] NCCL INFO Using network IB
e07r2n00:23102:24358 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.7.21<0>
e07r2n00:23102:24358 [2] NCCL INFO Using network IB
e07r2n02:4760:5858 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n01:6047:7323 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r1n18:15189:16547 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n00:24493:25710 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n04:1829:3075 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n17:31187:32200 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n02:4756:5860 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n14:22275:23497 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n09:23024:24184 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n13:31206:32207 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n01:627:1821 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n03:21866:23047 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n01:626:1823 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n03:21865:23050 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n00:24492:25711 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n04:1828:3077 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r1n18:15188:16551 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n01:6046:7322 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n10:29371:30326 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n11:16526:17682 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n04:1826:3076 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n13:31210:32209 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n02:4759:5859 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n13:31207:32206 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n02:4758:5857 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n08:30377:31697 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n04:1827:3073 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n03:21863:23045 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n13:31209:32208 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n03:21864:23049 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n12:16038:17016 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n01:629:1819 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n09:23023:24187 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n12:16039:17014 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n01:628:1825 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n09:23026:24186 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n09:23025:24185 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n14:22277:23498 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n00:24491:25708 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n12:16037:17013 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n10:29373:30327 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n14:22278:23496 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n01:6048:7326 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n14:22276:23489 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n01:6049:7320 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n11:16527:17685 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n00:24490:25707 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n11:16528:17686 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n08:30378:31693 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n11:16529:17687 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n17:31186:32202 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n12:16035:17011 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n07:11608:12938 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r1n18:15185:16349 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n06:26609:27563 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n02:4760:5858 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n17:31184:32198 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r1n18:15187:16550 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n06:26611:27566 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n18:25918:26911 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n06:26608:27568 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n18:25917:26912 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n07:11607:12941 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n17:31185:32201 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n06:26610:27569 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n10:29372:30328 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n10:29369:30329 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n08:30379:31736 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n08:30380:31690 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n02:4760:5858 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n07:11610:12943 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n07:11609:12944 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r3n01:6047:7323 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n00:24493:25710 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r1n18:15189:16547 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n04:1829:3075 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n02:4756:5860 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n18:25920:26909 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n17:31187:32200 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n18:25919:26914 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n14:22275:23497 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n01:6047:7323 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r1n18:15189:16547 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r3n00:24493:25710 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n09:23024:24184 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n13:31206:32207 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n02:4756:5860 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n04:1829:3075 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r3n00:24492:25711 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n17:31187:32200 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n14:22275:23497 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n13:31206:32207 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n02:4759:5859 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n09:23024:24184 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r1n18:15188:16551 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n02:4758:5857 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n04:1828:3077 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n01:6046:7322 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n11:16526:17682 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n04:1826:3076 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n00:24492:25711 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n04:1827:3073 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n00:24491:25708 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n13:31210:32209 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n12:16038:17016 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n09:23023:24187 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n03:21866:23047 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n09:23026:24186 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n13:31207:32206 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n02:4759:5859 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r1n18:15188:16551 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n09:23025:24185 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n13:31209:32208 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n02:4758:5857 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n03:21865:23050 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n00:24490:25707 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n04:1828:3077 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n03:21864:23049 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n04:1826:3076 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n14:22277:23498 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n01:6046:7322 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n11:16526:17682 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n12:16039:17014 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n01:627:1821 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r1n18:15185:16349 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n03:21863:23045 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n04:1827:3073 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n14:22278:23496 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n01:6049:7320 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n12:16037:17013 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n01:626:1823 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n00:24491:25708 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n13:31210:32209 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n09:23023:24187 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r1n18:15187:16550 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n03:21866:23047 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n14:22276:23489 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n01:6048:7326 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n12:16038:17016 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n01:629:1819 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r3n00:24490:25707 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n13:31207:32206 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n09:23026:24186 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n17:31186:32202 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n11:16527:17685 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n07:11608:12938 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r1n18:15185:16349 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n03:21865:23050 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n14:22277:23498 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r3n01:6049:7320 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n12:16035:17011 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n01:628:1825 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n13:31209:32208 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n09:23025:24185 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n17:31184:32198 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n11:16528:17686 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n07:11607:12941 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n06:26609:27563 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n10:29371:30326 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n08:30377:31697 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r1n18:15187:16550 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n03:21864:23049 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n14:22278:23496 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r3n01:6048:7326 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n12:16039:17014 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n01:627:1821 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n17:31185:32201 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n11:16529:17687 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n07:11608:12938 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n06:26611:27566 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n10:29373:30327 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n08:30378:31693 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n18:25918:26911 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n03:21863:23045 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n14:22276:23489 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n12:16037:17013 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n01:626:1823 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n17:31186:32202 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n11:16527:17685 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n07:11610:12943 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n06:26608:27568 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n10:29372:30328 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n08:30377:31697 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n18:25917:26912 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n12:16035:17011 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n01:629:1819 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n17:31184:32198 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n11:16528:17686 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n07:11609:12944 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n06:26610:27569 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n10:29369:30329 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n08:30378:31693 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n18:25918:26911 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n01:628:1825 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n17:31185:32201 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n11:16529:17687 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n07:11607:12941 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n06:26609:27563 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n10:29371:30326 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n08:30380:31690 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n18:25917:26912 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n07:11610:12943 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n06:26611:27566 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n10:29372:30328 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n08:30379:31736 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n18:25920:26909 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n07:11609:12944 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n06:26608:27568 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n10:29373:30327 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n08:30380:31690 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n18:25919:26914 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n06:26610:27569 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n10:29369:30329 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n08:30379:31736 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n18:25920:26909 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n18:25919:26914 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r1n19:30602:31679 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n19:2111:3520 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r1n19:30601:31683 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n19:2107:3519 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n19:2109:3517 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r1n19:30599:31677 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r1n19:30600:31684 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n00:23103:24289 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n00:23101:24353 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n19:2110:3521 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n15:28147:29326 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n15:28150:29329 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n15:28145:29327 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n15:28148:29328 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n00:23099:24278 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n00:23102:24358 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r1n19:30602:31679 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n19:2111:3520 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r1n19:30602:31679 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r1n19:30601:31683 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n19:2107:3519 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r1n19:30599:31677 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r1n19:30600:31684 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n19:2111:3520 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n19:2109:3517 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n19:2110:3521 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r1n19:30601:31683 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n00:23103:24289 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n19:2107:3519 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r1n19:30599:31677 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n00:23101:24353 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n19:2109:3517 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r1n19:30600:31684 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n15:28147:29326 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n15:28150:29329 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n19:2110:3521 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n15:28145:29327 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n00:23103:24289 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n15:28148:29328 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n00:23101:24353 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n15:28147:29326 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n15:28150:29329 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n15:28145:29327 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n00:23099:24278 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n15:28148:29328 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n00:23102:24358 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n16:26997:28160 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n05:909:2102 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n00:23099:24278 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n00:23102:24358 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n16:26998:28161 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n16:26996:28158 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n16:26995:28159 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n05:908:2101 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n05:907:2100 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n05:906:2099 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e07r2n16:26997:28160 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n05:909:2102 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n16:26998:28161 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n16:26997:28160 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n16:26996:28158 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n16:26995:28159 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n05:909:2102 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n16:26998:28161 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n16:26996:28158 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n16:26995:28159 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n05:908:2101 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n05:907:2100 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n05:906:2099 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e07r2n05:908:2101 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n05:907:2100 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n05:906:2099 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n07:11608:12938 [1] NCCL INFO Trees [0] 38/-1/-1->37->36 [1] 38/40/-1->37->36 comm 0x2aeaa8000ab0 nRanks 96 busId 26000
e07r2n07:11609:12944 [2] NCCL INFO Trees [0] 39/-1/-1->38->37 [1] 39/-1/-1->38->37 comm 0x2aca80000ab0 nRanks 96 busId 43000
e07r2n05:907:2100 [1] NCCL INFO Trees [0] 30/-1/-1->29->28 [1] 30/60/-1->29->28 comm 0x2af6c0000ab0 nRanks 96 busId 26000
e07r2n06:26608:27568 [0] NCCL INFO Trees [0] 33/48/-1->32->65 [1] 33/-1/-1->32->36 comm 0x2b2990000ab0 nRanks 96 busId 4000
e07r2n05:909:2102 [3] NCCL INFO Trees [0] -1/-1/-1->31->30 [1] -1/-1/-1->31->30 comm 0x2ac2e4000ab0 nRanks 96 busId 63000
e07r2n05:908:2101 [2] NCCL INFO Trees [0] 31/-1/-1->30->29 [1] 31/-1/-1->30->29 comm 0x2b0160000ab0 nRanks 96 busId 43000
e07r2n06:26609:27563 [1] NCCL INFO Trees [0] 34/16/-1->33->32 [1] 34/-1/-1->33->32 comm 0x2ae710000ab0 nRanks 96 busId 26000
e07r2n07:11607:12941 [0] NCCL INFO Trees [0] 37/-1/-1->36->41 [1] 37/32/-1->36->44 comm 0x2ac774000ab0 nRanks 96 busId 4000
e07r2n08:30378:31693 [1] NCCL INFO Trees [0] 42/36/-1->41->40 [1] 42/-1/-1->41->40 comm 0x2b2bac000ab0 nRanks 96 busId 26000
e07r2n07:11610:12943 [3] NCCL INFO Trees [0] -1/-1/-1->39->38 [1] -1/-1/-1->39->38 comm 0x2afd74000ab0 nRanks 96 busId 63000
e07r2n08:30377:31697 [0] NCCL INFO Trees [0] 41/44/-1->40->49 [1] 41/-1/-1->40->37 comm 0x2b4008000ab0 nRanks 96 busId 4000
e07r2n08:30380:31690 [3] NCCL INFO Trees [0] -1/-1/-1->43->42 [1] -1/-1/-1->43->42 comm 0x2b85a8000ab0 nRanks 96 busId 63000
e07r2n06:26611:27566 [3] NCCL INFO Trees [0] -1/-1/-1->35->34 [1] -1/-1/-1->35->34 comm 0x2b6078000ab0 nRanks 96 busId 63000
e07r2n09:23025:24185 [2] NCCL INFO Trees [0] 47/-1/-1->46->45 [1] 47/-1/-1->46->45 comm 0x2b44d8000ab0 nRanks 96 busId 43000
e07r2n08:30379:31736 [2] NCCL INFO Trees [0] 43/-1/-1->42->41 [1] 43/-1/-1->42->41 comm 0x2add50000ab0 nRanks 96 busId 43000
e07r2n06:26610:27569 [2] NCCL INFO Trees [0] 35/-1/-1->34->33 [1] 35/-1/-1->34->33 comm 0x2b9324000ab0 nRanks 96 busId 43000
e07r2n09:23024:24184 [1] NCCL INFO Trees [0] 46/-1/-1->45->44 [1] 46/52/-1->45->44 comm 0x2b39f0000ab0 nRanks 96 busId 26000
e07r2n05:906:2099 [0] NCCL INFO Trees [0] 29/-1/-1->28->24 [1] 29/12/-1->28->92 comm 0x2ba3ac000ab0 nRanks 96 busId 4000
e07r2n09:23023:24187 [0] NCCL INFO Trees [0] 45/-1/-1->44->40 [1] 45/36/-1->44->60 comm 0x2af760000ab0 nRanks 96 busId 4000
e07r2n09:23026:24186 [3] NCCL INFO Trees [0] -1/-1/-1->47->46 [1] -1/-1/-1->47->46 comm 0x2b48e0000ab0 nRanks 96 busId 63000
e07r2n04:1829:3075 [3] NCCL INFO Trees [0] -1/-1/-1->27->26 [1] -1/-1/-1->27->26 comm 0x2b663c000ab0 nRanks 96 busId 63000
e07r2n10:29369:30329 [0] NCCL INFO Trees [0] 49/56/-1->48->32 [1] 49/-1/-1->48->52 comm 0x2b24f0000ab0 nRanks 96 busId 4000
e07r2n10:29372:30328 [2] NCCL INFO Trees [0] 51/-1/-1->50->49 [1] 51/-1/-1->50->49 comm 0x2b4700000ab0 nRanks 96 busId 43000
e07r2n04:1828:3077 [2] NCCL INFO Trees [0] 27/-1/-1->26->25 [1] 27/-1/-1->26->25 comm 0x2b200c000ab0 nRanks 96 busId 43000
e07r2n10:29371:30326 [1] NCCL INFO Trees [0] 50/40/-1->49->48 [1] 50/-1/-1->49->48 comm 0x2ad4c4000ab0 nRanks 96 busId 26000
e07r2n04:1827:3073 [1] NCCL INFO Trees [0] 26/20/-1->25->24 [1] 26/-1/-1->25->24 comm 0x2abfb4000ab0 nRanks 96 busId 26000
e07r2n10:29373:30327 [3] NCCL INFO Trees [0] -1/-1/-1->51->50 [1] -1/-1/-1->51->50 comm 0x2b2b38000ab0 nRanks 96 busId 63000
e07r2n04:1826:3076 [0] NCCL INFO Trees [0] 25/28/-1->24->16 [1] 25/-1/-1->24->21 comm 0x2b09f8000ab0 nRanks 96 busId 4000
e07r2n03:21866:23047 [3] NCCL INFO Trees [0] -1/-1/-1->23->22 [1] -1/-1/-1->23->22 comm 0x2b6bd4000ab0 nRanks 96 busId 63000
e07r2n12:16039:17014 [3] NCCL INFO Trees [0] -1/-1/-1->59->58 [1] -1/-1/-1->59->58 comm 0x2b3ec8000ab0 nRanks 96 busId 63000
e07r2n14:22276:23489 [1] NCCL INFO Trees [0] 66/32/-1->65->64 [1] 66/-1/-1->65->64 comm 0x2aee9c000ab0 nRanks 96 busId 26000
e07r2n11:16529:17687 [3] NCCL INFO Trees [0] -1/-1/-1->55->54 [1] -1/-1/-1->55->54 comm 0x2b82f0000ab0 nRanks 96 busId 63000
e07r2n13:31206:32207 [0] NCCL INFO Trees [0] 61/-1/-1->60->56 [1] 61/44/-1->60->29 comm 0x2ae9c8000ab0 nRanks 96 busId 4000
e07r2n14:22277:23498 [2] NCCL INFO Trees [0] 67/-1/-1->66->65 [1] 67/-1/-1->66->65 comm 0x2b736c000ab0 nRanks 96 busId 43000
e07r2n11:16528:17686 [2] NCCL INFO Trees [0] 55/-1/-1->54->53 [1] 55/-1/-1->54->53 comm 0x2b8a80000ab0 nRanks 96 busId 43000
e07r2n13:31207:32206 [1] NCCL INFO Trees [0] 62/-1/-1->61->60 [1] 62/76/-1->61->60 comm 0x2aeb40000ab0 nRanks 96 busId 26000
e07r2n15:28145:29327 [0] NCCL INFO Trees [0] 69/-1/-1->68->73 [1] 69/64/-1->68->76 comm 0x2b1e30000ab0 nRanks 96 busId 4000
e07r2n14:22275:23497 [0] NCCL INFO Trees [0] 65/80/-1->64->0 [1] 65/-1/-1->64->68 comm 0x2b696c000ab0 nRanks 96 busId 4000
e07r2n11:16527:17685 [1] NCCL INFO Trees [0] 54/-1/-1->53->52 [1] 54/56/-1->53->52 comm 0x2ae98c000ab0 nRanks 96 busId 26000
e07r2n13:31210:32209 [3] NCCL INFO Trees [0] -1/-1/-1->63->62 [1] -1/-1/-1->63->62 comm 0x2af840000ab0 nRanks 96 busId 63000
e07r2n15:28147:29326 [1] NCCL INFO Trees [0] 70/-1/-1->69->68 [1] 70/72/-1->69->68 comm 0x2b19d0000ab0 nRanks 96 busId 26000
e07r2n12:16038:17016 [2] NCCL INFO Trees [0] 59/-1/-1->58->57 [1] 59/-1/-1->58->57 comm 0x2b93f8000ab0 nRanks 96 busId 43000
e07r2n14:22278:23496 [3] NCCL INFO Trees [0] -1/-1/-1->67->66 [1] -1/-1/-1->67->66 comm 0x2b1038000ab0 nRanks 96 busId 63000
e07r2n11:16526:17682 [0] NCCL INFO Trees [0] 53/-1/-1->52->57 [1] 53/48/-1->52->45 comm 0x2b8468000ab0 nRanks 96 busId 4000
e07r2n13:31209:32208 [2] NCCL INFO Trees [0] 63/-1/-1->62->61 [1] 63/-1/-1->62->61 comm 0x2ab7c4000ab0 nRanks 96 busId 43000
e07r2n15:28148:29328 [2] NCCL INFO Trees [0] 71/-1/-1->70->69 [1] 71/-1/-1->70->69 comm 0x2afa50000ab0 nRanks 96 busId 43000
e07r2n12:16037:17013 [1] NCCL INFO Trees [0] 58/52/-1->57->56 [1] 58/-1/-1->57->56 comm 0x2ae0f4000ab0 nRanks 96 busId 26000
e07r2n03:21865:23050 [2] NCCL INFO Trees [0] 23/-1/-1->22->21 [1] 23/-1/-1->22->21 comm 0x2b601c000ab0 nRanks 96 busId 43000
e07r2n06:26608:27568 [0] NCCL INFO Channel 00/0 : 31[63000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2990000ab0 nRanks 96
e07r3n00:24493:25710 [3] NCCL INFO Trees [0] -1/-1/-1->91->90 [1] -1/-1/-1->91->90 comm 0x2b061c000ab0 nRanks 96 busId 63000
e07r2n02:4759:5859 [2] NCCL INFO Trees [0] 19/-1/-1->18->17 [1] 19/-1/-1->18->17 comm 0x2b32b8000ab0 nRanks 96 busId 43000
e07r1n18:15185:16349 [0] NCCL INFO Channel 00/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
e07r1n18:15185:16349 [0] NCCL INFO Channel 01/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
e07r1n18:15185:16349 [0] NCCL INFO Trees [0] 1/64/-1->0->-1 [1] 1/-1/-1->0->4 comm 0x2b2c88000ab0 nRanks 96 busId 4000
e07r2n15:28150:29329 [3] NCCL INFO Trees [0] -1/-1/-1->71->70 [1] -1/-1/-1->71->70 comm 0x2b3a44000ab0 nRanks 96 busId 63000
e07r2n12:16035:17011 [0] NCCL INFO Trees [0] 57/60/-1->56->48 [1] 57/-1/-1->56->53 comm 0x2b831c000ab0 nRanks 96 busId 4000
e07r2n03:21864:23049 [1] NCCL INFO Trees [0] 22/-1/-1->21->20 [1] 22/24/-1->21->20 comm 0x2b3270000ab0 nRanks 96 busId 26000
e07r3n00:24492:25711 [2] NCCL INFO Trees [0] 91/-1/-1->90->89 [1] 91/-1/-1->90->89 comm 0x2b96c4000ab0 nRanks 96 busId 43000
e07r2n02:4760:5858 [3] NCCL INFO Trees [0] -1/-1/-1->19->18 [1] -1/-1/-1->19->18 comm 0x2b72e8000ab0 nRanks 96 busId 63000
e07r2n08:30377:31697 [0] NCCL INFO Channel 00/0 : 39[63000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4008000ab0 nRanks 96
e07r2n19:2110:3521 [2] NCCL INFO Trees [0] 87/-1/-1->86->85 [1] 87/-1/-1->86->85 comm 0x2b9ce8000ab0 nRanks 96 busId 43000
e07r2n16:26995:28159 [0] NCCL INFO Trees [0] 73/76/-1->72->81 [1] 73/-1/-1->72->69 comm 0x2b7c84000ab0 nRanks 96 busId 4000
e07r2n01:629:1819 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] -1/-1/-1->15->14 comm 0x2ace88000ab0 nRanks 96 busId 63000
e07r2n17:31185:32201 [1] NCCL INFO Trees [0] 78/-1/-1->77->76 [1] 78/84/-1->77->76 comm 0x2ac6ec000ab0 nRanks 96 busId 26000
e07r2n18:25920:26909 [3] NCCL INFO Trees [0] -1/-1/-1->83->82 [1] -1/-1/-1->83->82 comm 0x2af5b8000ab0 nRanks 96 busId 63000
e07r2n07:11607:12941 [0] NCCL INFO Channel 00/0 : 35[63000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac774000ab0 nRanks 96
e07r1n18:15187:16550 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 comm 0x2ac740000ab0 nRanks 96 busId 26000
e07r1n18:15188:16551 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 comm 0x2b6a08000ab0 nRanks 96 busId 43000
e07r1n18:15189:16547 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 comm 0x2b6d50000ab0 nRanks 96 busId 63000
e07r2n03:21863:23045 [0] NCCL INFO Trees [0] 21/-1/-1->20->25 [1] 21/16/-1->20->13 comm 0x2b3b20000ab0 nRanks 96 busId 4000
e07r3n00:24490:25707 [0] NCCL INFO Trees [0] 89/92/-1->88->80 [1] 89/-1/-1->88->85 comm 0x2b84fc000ab0 nRanks 96 busId 4000
e07r2n02:4758:5857 [1] NCCL INFO Trees [0] 18/8/-1->17->16 [1] 18/-1/-1->17->16 comm 0x2b8424000ab0 nRanks 96 busId 26000
e07r2n08:30379:31736 [2] NCCL INFO Channel 00 : 42[43000] -> 43[63000] via P2P/IPC comm 0x2add50000ab0 nRanks 96
e07r2n19:2111:3520 [3] NCCL INFO Trees [0] -1/-1/-1->87->86 [1] -1/-1/-1->87->86 comm 0x2b30c8000ab0 nRanks 96 busId 63000
e07r2n16:26997:28160 [2] NCCL INFO Trees [0] 75/-1/-1->74->73 [1] 75/-1/-1->74->73 comm 0x2b2168000ab0 nRanks 96 busId 43000
e07r2n01:627:1821 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/20/-1->13->12 comm 0x2b75e4000ab0 nRanks 96 busId 26000
e07r2n17:31184:32198 [0] NCCL INFO Trees [0] 77/-1/-1->76->72 [1] 77/68/-1->76->61 comm 0x2baf30000ab0 nRanks 96 busId 4000
e07r2n18:25917:26912 [0] NCCL INFO Trees [0] 81/88/-1->80->64 [1] 81/-1/-1->80->84 comm 0x2b9534000ab0 nRanks 96 busId 4000
e07r2n07:11609:12944 [2] NCCL INFO Channel 00 : 38[43000] -> 39[63000] via P2P/IPC comm 0x2aca80000ab0 nRanks 96
e07r2n05:907:2100 [1] NCCL INFO Channel 00 : 29[26000] -> 30[43000] via P2P/IPC comm 0x2af6c0000ab0 nRanks 96
e07r1n19:30602:31679 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 comm 0x2b2b9c000ab0 nRanks 96 busId 63000
e07r3n01:6047:7323 [1] NCCL INFO Trees [0] 94/-1/-1->93->92 [1] 94/-1/-1->93->92 comm 0x2adc2c000ab0 nRanks 96 busId 26000
e07r2n00:23103:24289 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] -1/-1/-1->11->10 comm 0x2b8398000ab0 nRanks 96 busId 63000
e07r2n06:26609:27563 [1] NCCL INFO Channel 00 : 33[26000] -> 34[43000] via P2P/IPC comm 0x2ae710000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 00/0 : 43[63000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af760000ab0 nRanks 96
e07r3n00:24491:25708 [1] NCCL INFO Trees [0] 90/84/-1->89->88 [1] 90/-1/-1->89->88 comm 0x2b9b5c000ab0 nRanks 96 busId 26000
e07r2n02:4756:5860 [0] NCCL INFO Trees [0] 17/24/-1->16->33 [1] 17/-1/-1->16->20 comm 0x2b6f54000ab0 nRanks 96 busId 4000
e07r2n08:30378:31693 [1] NCCL INFO Channel 00 : 41[26000] -> 42[43000] via P2P/IPC comm 0x2b2bac000ab0 nRanks 96
e07r2n19:2109:3517 [1] NCCL INFO Trees [0] 86/-1/-1->85->84 [1] 86/88/-1->85->84 comm 0x2b2e10000ab0 nRanks 96 busId 26000
e07r2n16:26996:28158 [1] NCCL INFO Trees [0] 74/68/-1->73->72 [1] 74/-1/-1->73->72 comm 0x2ade8c000ab0 nRanks 96 busId 26000
e07r2n01:628:1825 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 comm 0x2ae908000ab0 nRanks 96 busId 43000
e07r2n17:31186:32202 [2] NCCL INFO Trees [0] 79/-1/-1->78->77 [1] 79/-1/-1->78->77 comm 0x2b2e7c000ab0 nRanks 96 busId 43000
e07r2n18:25919:26914 [2] NCCL INFO Trees [0] 83/-1/-1->82->81 [1] 83/-1/-1->82->81 comm 0x2aef38000ab0 nRanks 96 busId 43000
e07r2n07:11609:12944 [2] NCCL INFO Channel 01 : 38[43000] -> 39[63000] via P2P/IPC comm 0x2aca80000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 00/0 : 27[63000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba3ac000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Trees [0] 5/-1/-1->4->9 [1] 5/0/-1->4->12 comm 0x2b6e1c000ab0 nRanks 96 busId 4000
e07r3n01:6046:7322 [0] NCCL INFO Trees [0] 93/-1/-1->92->88 [1] 93/28/-1->92->-1 comm 0x2b0b68000ab0 nRanks 96 busId 4000
e07r2n00:23101:24353 [1] NCCL INFO Trees [0] 10/4/-1->9->8 [1] 10/-1/-1->9->8 comm 0x2abe94000ab0 nRanks 96 busId 26000
e07r2n06:26610:27569 [2] NCCL INFO Channel 00 : 34[43000] -> 35[63000] via P2P/IPC comm 0x2b9324000ab0 nRanks 96
e07r2n09:23024:24184 [1] NCCL INFO Channel 00 : 45[26000] -> 46[43000] via P2P/IPC comm 0x2b39f0000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 00/0 : 47[63000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b24f0000ab0 nRanks 96
e07r2n08:30378:31693 [1] NCCL INFO Channel 01 : 41[26000] -> 42[43000] via P2P/IPC comm 0x2b2bac000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Trees [0] 85/-1/-1->84->89 [1] 85/80/-1->84->77 comm 0x2b5860000ab0 nRanks 96 busId 4000
e07r2n16:26998:28161 [3] NCCL INFO Trees [0] -1/-1/-1->75->74 [1] -1/-1/-1->75->74 comm 0x2b5c98000ab0 nRanks 96 busId 63000
e07r2n01:626:1823 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] 13/4/-1->12->28 comm 0x2ae748000ab0 nRanks 96 busId 4000
e07r2n17:31187:32200 [3] NCCL INFO Trees [0] -1/-1/-1->79->78 [1] -1/-1/-1->79->78 comm 0x2b1110000ab0 nRanks 96 busId 63000
e07r2n18:25918:26911 [1] NCCL INFO Trees [0] 82/72/-1->81->80 [1] 82/-1/-1->81->80 comm 0x2b094c000ab0 nRanks 96 busId 26000
e07r2n07:11608:12938 [1] NCCL INFO Channel 00 : 37[26000] -> 38[43000] via P2P/IPC comm 0x2aeaa8000ab0 nRanks 96
e07r2n05:908:2101 [2] NCCL INFO Channel 00 : 30[43000] -> 31[63000] via P2P/IPC comm 0x2b0160000ab0 nRanks 96
e07r1n19:30600:31684 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/8/-1->5->4 comm 0x2b4630000ab0 nRanks 96 busId 26000
e07r3n01:6048:7326 [2] NCCL INFO Trees [0] 95/-1/-1->94->93 [1] 95/-1/-1->94->93 comm 0x2b3f2c000ab0 nRanks 96 busId 43000
e07r2n00:23099:24278 [0] NCCL INFO Trees [0] 9/12/-1->8->17 [1] 9/-1/-1->8->5 comm 0x2b7be0000ab0 nRanks 96 busId 4000
e07r2n06:26609:27563 [1] NCCL INFO Channel 01 : 33[26000] -> 34[43000] via P2P/IPC comm 0x2ae710000ab0 nRanks 96
e07r2n09:23025:24185 [2] NCCL INFO Channel 00 : 46[43000] -> 47[63000] via P2P/IPC comm 0x2b44d8000ab0 nRanks 96
e07r2n10:29371:30326 [1] NCCL INFO Channel 00 : 49[26000] -> 50[43000] via P2P/IPC comm 0x2ad4c4000ab0 nRanks 96
e07r2n04:1828:3077 [2] NCCL INFO Channel 00 : 26[43000] -> 27[63000] via P2P/IPC comm 0x2b200c000ab0 nRanks 96
e07r2n08:30379:31736 [2] NCCL INFO Channel 01 : 42[43000] -> 43[63000] via P2P/IPC comm 0x2add50000ab0 nRanks 96
e07r2n07:11608:12938 [1] NCCL INFO Channel 01 : 37[26000] -> 38[43000] via P2P/IPC comm 0x2aeaa8000ab0 nRanks 96
e07r2n05:907:2100 [1] NCCL INFO Channel 01 : 29[26000] -> 30[43000] via P2P/IPC comm 0x2af6c0000ab0 nRanks 96
e07r1n19:30601:31683 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 comm 0x2b9508000ab0 nRanks 96 busId 43000
e07r3n01:6049:7320 [3] NCCL INFO Trees [0] -1/-1/-1->95->94 [1] -1/-1/-1->95->94 comm 0x2b4e20000ab0 nRanks 96 busId 63000
e07r2n00:23102:24358 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 comm 0x2acbf4000ab0 nRanks 96 busId 43000
e07r2n06:26610:27569 [2] NCCL INFO Channel 01 : 34[43000] -> 35[63000] via P2P/IPC comm 0x2b9324000ab0 nRanks 96
e07r2n09:23024:24184 [1] NCCL INFO Channel 01 : 45[26000] -> 46[43000] via P2P/IPC comm 0x2b39f0000ab0 nRanks 96
e07r2n10:29372:30328 [2] NCCL INFO Channel 00 : 50[43000] -> 51[63000] via P2P/IPC comm 0x2b4700000ab0 nRanks 96
e07r2n04:1827:3073 [1] NCCL INFO Channel 00 : 25[26000] -> 26[43000] via P2P/IPC comm 0x2abfb4000ab0 nRanks 96
e07r2n05:908:2101 [2] NCCL INFO Channel 01 : 30[43000] -> 31[63000] via P2P/IPC comm 0x2b0160000ab0 nRanks 96
e07r2n09:23025:24185 [2] NCCL INFO Channel 01 : 46[43000] -> 47[63000] via P2P/IPC comm 0x2b44d8000ab0 nRanks 96
e07r2n10:29371:30326 [1] NCCL INFO Channel 01 : 49[26000] -> 50[43000] via P2P/IPC comm 0x2ad4c4000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 00/0 : 23[63000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b09f8000ab0 nRanks 96
e07r2n10:29372:30328 [2] NCCL INFO Channel 01 : 50[43000] -> 51[63000] via P2P/IPC comm 0x2b4700000ab0 nRanks 96
e07r2n04:1828:3077 [2] NCCL INFO Channel 01 : 26[43000] -> 27[63000] via P2P/IPC comm 0x2b200c000ab0 nRanks 96
e07r2n04:1827:3073 [1] NCCL INFO Channel 01 : 25[26000] -> 26[43000] via P2P/IPC comm 0x2abfb4000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 00/0 : 67[63000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b1e30000ab0 nRanks 96
e07r2n13:31207:32206 [1] NCCL INFO Channel 00 : 61[26000] -> 62[43000] via P2P/IPC comm 0x2aeb40000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 00/0 : 59[63000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ae9c8000ab0 nRanks 96
e07r2n14:22276:23489 [1] NCCL INFO Channel 00 : 65[26000] -> 66[43000] via P2P/IPC comm 0x2aee9c000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 00/0 : 63[63000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b696c000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 00/0 : 51[63000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b8468000ab0 nRanks 96
e07r2n14:22277:23498 [2] NCCL INFO Channel 00 : 66[43000] -> 67[63000] via P2P/IPC comm 0x2b736c000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 00/0 : 55[63000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b831c000ab0 nRanks 96
e07r2n03:21865:23050 [2] NCCL INFO Channel 00 : 22[43000] -> 23[63000] via P2P/IPC comm 0x2b601c000ab0 nRanks 96
e07r2n12:16038:17016 [2] NCCL INFO Channel 00 : 58[43000] -> 59[63000] via P2P/IPC comm 0x2b93f8000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 00/0 : 19[63000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3b20000ab0 nRanks 96
e07r2n12:16037:17013 [1] NCCL INFO Channel 00 : 57[26000] -> 58[43000] via P2P/IPC comm 0x2ae0f4000ab0 nRanks 96
e07r2n03:21864:23049 [1] NCCL INFO Channel 00 : 21[26000] -> 22[43000] via P2P/IPC comm 0x2b3270000ab0 nRanks 96
e07r2n15:28147:29326 [1] NCCL INFO Channel 00 : 69[26000] -> 70[43000] via P2P/IPC comm 0x2b19d0000ab0 nRanks 96
e07r2n13:31207:32206 [1] NCCL INFO Channel 01 : 61[26000] -> 62[43000] via P2P/IPC comm 0x2aeb40000ab0 nRanks 96
e07r2n11:16527:17685 [1] NCCL INFO Channel 00 : 53[26000] -> 54[43000] via P2P/IPC comm 0x2ae98c000ab0 nRanks 96
e07r1n18:15185:16349 [0] NCCL INFO Channel 00/0 : 95[63000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b2c88000ab0 nRanks 96
e07r2n12:16038:17016 [2] NCCL INFO Channel 01 : 58[43000] -> 59[63000] via P2P/IPC comm 0x2b93f8000ab0 nRanks 96
e07r2n15:28148:29328 [2] NCCL INFO Channel 00 : 70[43000] -> 71[63000] via P2P/IPC comm 0x2afa50000ab0 nRanks 96
e07r2n13:31209:32208 [2] NCCL INFO Channel 00 : 62[43000] -> 63[63000] via P2P/IPC comm 0x2ab7c4000ab0 nRanks 96
e07r2n11:16528:17686 [2] NCCL INFO Channel 00 : 54[43000] -> 55[63000] via P2P/IPC comm 0x2b8a80000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 00/0 : 83[63000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5860000ab0 nRanks 96
e07r2n02:4759:5859 [2] NCCL INFO Channel 00 : 18[43000] -> 19[63000] via P2P/IPC comm 0x2b32b8000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 00/0 : 75[63000] -> 76[4000] [receive] via NET/IB/0 comm 0x2baf30000ab0 nRanks 96
e07r3n00:24490:25707 [0] NCCL INFO Channel 00/0 : 87[63000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b84fc000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Channel 00/0 : 3[63000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b6e1c000ab0 nRanks 96
e07r1n18:15187:16550 [1] NCCL INFO Channel 00 : 1[26000] -> 2[43000] via P2P/IPC comm 0x2ac740000ab0 nRanks 96
e07r2n13:31209:32208 [2] NCCL INFO Channel 01 : 62[43000] -> 63[63000] via P2P/IPC comm 0x2ab7c4000ab0 nRanks 96
e07r2n11:16527:17685 [1] NCCL INFO Channel 01 : 53[26000] -> 54[43000] via P2P/IPC comm 0x2ae98c000ab0 nRanks 96
e07r2n19:2110:3521 [2] NCCL INFO Channel 00 : 86[43000] -> 87[63000] via P2P/IPC comm 0x2b9ce8000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 00/0 : 15[63000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b6f54000ab0 nRanks 96
e07r3n00:24492:25711 [2] NCCL INFO Channel 00 : 90[43000] -> 91[63000] via P2P/IPC comm 0x2b96c4000ab0 nRanks 96
e07r2n14:22277:23498 [2] NCCL INFO Channel 01 : 66[43000] -> 67[63000] via P2P/IPC comm 0x2b736c000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 00/0 : 79[63000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b9534000ab0 nRanks 96
e07r2n16:26997:28160 [2] NCCL INFO Channel 00 : 74[43000] -> 75[63000] via P2P/IPC comm 0x2b2168000ab0 nRanks 96
e07r3n01:6046:7322 [0] NCCL INFO Channel 00/0 : 91[63000] -> 92[4000] [receive] via NET/IB/0 comm 0x2b0b68000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 00/0 : 7[63000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7be0000ab0 nRanks 96
e07r2n01:628:1825 [2] NCCL INFO Channel 00 : 14[43000] -> 15[63000] via P2P/IPC comm 0x2ae908000ab0 nRanks 96
e07r2n03:21864:23049 [1] NCCL INFO Channel 01 : 21[26000] -> 22[43000] via P2P/IPC comm 0x2b3270000ab0 nRanks 96
e07r1n18:15188:16551 [2] NCCL INFO Channel 00 : 2[43000] -> 3[63000] via P2P/IPC comm 0x2b6a08000ab0 nRanks 96
e07r2n11:16528:17686 [2] NCCL INFO Channel 01 : 54[43000] -> 55[63000] via P2P/IPC comm 0x2b8a80000ab0 nRanks 96
e07r2n19:2109:3517 [1] NCCL INFO Channel 00 : 85[26000] -> 86[43000] via P2P/IPC comm 0x2b2e10000ab0 nRanks 96
e07r2n02:4759:5859 [2] NCCL INFO Channel 01 : 18[43000] -> 19[63000] via P2P/IPC comm 0x2b32b8000ab0 nRanks 96
e07r3n00:24491:25708 [1] NCCL INFO Channel 00 : 89[26000] -> 90[43000] via P2P/IPC comm 0x2b9b5c000ab0 nRanks 96
e07r2n14:22276:23489 [1] NCCL INFO Channel 01 : 65[26000] -> 66[43000] via P2P/IPC comm 0x2aee9c000ab0 nRanks 96
e07r2n18:25918:26911 [1] NCCL INFO Channel 00 : 81[26000] -> 82[43000] via P2P/IPC comm 0x2b094c000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 00/0 : 71[63000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7c84000ab0 nRanks 96
e07r3n01:6047:7323 [1] NCCL INFO Channel 00 : 93[26000] -> 94[43000] via P2P/IPC comm 0x2adc2c000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 00/0 : 11[63000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae748000ab0 nRanks 96
e07r2n03:21865:23050 [2] NCCL INFO Channel 01 : 22[43000] -> 23[63000] via P2P/IPC comm 0x2b601c000ab0 nRanks 96
e07r2n15:28147:29326 [1] NCCL INFO Channel 01 : 69[26000] -> 70[43000] via P2P/IPC comm 0x2b19d0000ab0 nRanks 96
e07r2n12:16037:17013 [1] NCCL INFO Channel 01 : 57[26000] -> 58[43000] via P2P/IPC comm 0x2ae0f4000ab0 nRanks 96
e07r1n18:15187:16550 [1] NCCL INFO Channel 01 : 1[26000] -> 2[43000] via P2P/IPC comm 0x2ac740000ab0 nRanks 96
e07r1n18:15188:16551 [2] NCCL INFO Channel 01 : 2[43000] -> 3[63000] via P2P/IPC comm 0x2b6a08000ab0 nRanks 96
e07r2n19:2110:3521 [2] NCCL INFO Channel 01 : 86[43000] -> 87[63000] via P2P/IPC comm 0x2b9ce8000ab0 nRanks 96
e07r2n02:4758:5857 [1] NCCL INFO Channel 00 : 17[26000] -> 18[43000] via P2P/IPC comm 0x2b8424000ab0 nRanks 96
e07r3n00:24491:25708 [1] NCCL INFO Channel 01 : 89[26000] -> 90[43000] via P2P/IPC comm 0x2b9b5c000ab0 nRanks 96
e07r2n18:25919:26914 [2] NCCL INFO Channel 00 : 82[43000] -> 83[63000] via P2P/IPC comm 0x2aef38000ab0 nRanks 96
e07r2n16:26996:28158 [1] NCCL INFO Channel 00 : 73[26000] -> 74[43000] via P2P/IPC comm 0x2ade8c000ab0 nRanks 96
e07r3n01:6048:7326 [2] NCCL INFO Channel 00 : 94[43000] -> 95[63000] via P2P/IPC comm 0x2b3f2c000ab0 nRanks 96
e07r2n01:627:1821 [1] NCCL INFO Channel 00 : 13[26000] -> 14[43000] via P2P/IPC comm 0x2b75e4000ab0 nRanks 96
e07r2n15:28148:29328 [2] NCCL INFO Channel 01 : 70[43000] -> 71[63000] via P2P/IPC comm 0x2afa50000ab0 nRanks 96
e07r2n00:23101:24353 [1] NCCL INFO Channel 00 : 9[26000] -> 10[43000] via P2P/IPC comm 0x2abe94000ab0 nRanks 96
e07r1n19:30601:31683 [2] NCCL INFO Channel 00 : 6[43000] -> 7[63000] via P2P/IPC comm 0x2b9508000ab0 nRanks 96
e07r2n19:2109:3517 [1] NCCL INFO Channel 01 : 85[26000] -> 86[43000] via P2P/IPC comm 0x2b2e10000ab0 nRanks 96
e07r2n02:4758:5857 [1] NCCL INFO Channel 01 : 17[26000] -> 18[43000] via P2P/IPC comm 0x2b8424000ab0 nRanks 96
e07r3n00:24492:25711 [2] NCCL INFO Channel 01 : 90[43000] -> 91[63000] via P2P/IPC comm 0x2b96c4000ab0 nRanks 96
e07r2n18:25918:26911 [1] NCCL INFO Channel 01 : 81[26000] -> 82[43000] via P2P/IPC comm 0x2b094c000ab0 nRanks 96
e07r2n16:26997:28160 [2] NCCL INFO Channel 01 : 74[43000] -> 75[63000] via P2P/IPC comm 0x2b2168000ab0 nRanks 96
e07r3n01:6047:7323 [1] NCCL INFO Channel 01 : 93[26000] -> 94[43000] via P2P/IPC comm 0x2adc2c000ab0 nRanks 96
e07r2n01:628:1825 [2] NCCL INFO Channel 01 : 14[43000] -> 15[63000] via P2P/IPC comm 0x2ae908000ab0 nRanks 96
e07r2n00:23102:24358 [2] NCCL INFO Channel 00 : 10[43000] -> 11[63000] via P2P/IPC comm 0x2acbf4000ab0 nRanks 96
e07r1n19:30600:31684 [1] NCCL INFO Channel 00 : 5[26000] -> 6[43000] via P2P/IPC comm 0x2b4630000ab0 nRanks 96
e07r2n18:25919:26914 [2] NCCL INFO Channel 01 : 82[43000] -> 83[63000] via P2P/IPC comm 0x2aef38000ab0 nRanks 96
e07r2n16:26996:28158 [1] NCCL INFO Channel 01 : 73[26000] -> 74[43000] via P2P/IPC comm 0x2ade8c000ab0 nRanks 96
e07r3n01:6048:7326 [2] NCCL INFO Channel 01 : 94[43000] -> 95[63000] via P2P/IPC comm 0x2b3f2c000ab0 nRanks 96
e07r2n01:627:1821 [1] NCCL INFO Channel 01 : 13[26000] -> 14[43000] via P2P/IPC comm 0x2b75e4000ab0 nRanks 96
e07r2n00:23101:24353 [1] NCCL INFO Channel 01 : 9[26000] -> 10[43000] via P2P/IPC comm 0x2abe94000ab0 nRanks 96
e07r1n19:30600:31684 [1] NCCL INFO Channel 01 : 5[26000] -> 6[43000] via P2P/IPC comm 0x2b4630000ab0 nRanks 96
e07r2n00:23102:24358 [2] NCCL INFO Channel 01 : 10[43000] -> 11[63000] via P2P/IPC comm 0x2acbf4000ab0 nRanks 96
e07r1n19:30601:31683 [2] NCCL INFO Channel 01 : 6[43000] -> 7[63000] via P2P/IPC comm 0x2b9508000ab0 nRanks 96
e07r2n17:31185:32201 [1] NCCL INFO Channel 00 : 77[26000] -> 78[43000] via P2P/IPC comm 0x2ac6ec000ab0 nRanks 96
e07r2n17:31186:32202 [2] NCCL INFO Channel 00 : 78[43000] -> 79[63000] via P2P/IPC comm 0x2b2e7c000ab0 nRanks 96
e07r2n17:31185:32201 [1] NCCL INFO Channel 01 : 77[26000] -> 78[43000] via P2P/IPC comm 0x2ac6ec000ab0 nRanks 96
e07r2n17:31186:32202 [2] NCCL INFO Channel 01 : 78[43000] -> 79[63000] via P2P/IPC comm 0x2b2e7c000ab0 nRanks 96
e07r2n05:909:2102 [3] NCCL INFO Channel 00/0 : 31[63000] -> 32[4000] [send] via NET/IB/0 comm 0x2ac2e4000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 01/0 : 27[63000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba3ac000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 00 : 28[4000] -> 29[26000] via P2P/IPC comm 0x2ba3ac000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 01/0 : 31[63000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2990000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 00 : 32[4000] -> 33[26000] via P2P/IPC comm 0x2b2990000ab0 nRanks 96
e07r2n08:30380:31690 [3] NCCL INFO Channel 00/0 : 43[63000] -> 44[4000] [send] via NET/IB/0 comm 0x2b85a8000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 01/0 : 39[63000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4008000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 01 : 28[4000] -> 29[26000] via P2P/IPC comm 0x2ba3ac000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 00 : 40[4000] -> 41[26000] via P2P/IPC comm 0x2b4008000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 01 : 32[4000] -> 33[26000] via P2P/IPC comm 0x2b2990000ab0 nRanks 96
e07r2n04:1829:3075 [3] NCCL INFO Channel 00/0 : 27[63000] -> 28[4000] [send] via NET/IB/0 comm 0x2b663c000ab0 nRanks 96
e07r2n07:11610:12943 [3] NCCL INFO Channel 00/0 : 39[63000] -> 40[4000] [send] via NET/IB/0 comm 0x2afd74000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 01/0 : 35[63000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac774000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 00 : 36[4000] -> 37[26000] via P2P/IPC comm 0x2ac774000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 01 : 40[4000] -> 41[26000] via P2P/IPC comm 0x2b4008000ab0 nRanks 96
e07r2n09:23026:24186 [3] NCCL INFO Channel 00/0 : 47[63000] -> 48[4000] [send] via NET/IB/0 comm 0x2b48e0000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 01/0 : 43[63000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af760000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 00 : 44[4000] -> 45[26000] via P2P/IPC comm 0x2af760000ab0 nRanks 96
e07r2n10:29373:30327 [3] NCCL INFO Channel 00/0 : 51[63000] -> 52[4000] [send] via NET/IB/0 comm 0x2b2b38000ab0 nRanks 96
e07r2n06:26611:27566 [3] NCCL INFO Channel 00/0 : 35[63000] -> 36[4000] [send] via NET/IB/0 comm 0x2b6078000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 01 : 36[4000] -> 37[26000] via P2P/IPC comm 0x2ac774000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 01/0 : 47[63000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b24f0000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 00 : 48[4000] -> 49[26000] via P2P/IPC comm 0x2b24f0000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 01 : 44[4000] -> 45[26000] via P2P/IPC comm 0x2af760000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 01 : 48[4000] -> 49[26000] via P2P/IPC comm 0x2b24f0000ab0 nRanks 96
e07r2n03:21866:23047 [3] NCCL INFO Channel 00/0 : 23[63000] -> 24[4000] [send] via NET/IB/0 comm 0x2b6bd4000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 01/0 : 23[63000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b09f8000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 00 : 24[4000] -> 25[26000] via P2P/IPC comm 0x2b09f8000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 01/0 : 59[63000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ae9c8000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 00 : 60[4000] -> 61[26000] via P2P/IPC comm 0x2ae9c8000ab0 nRanks 96
e07r2n11:16529:17687 [3] NCCL INFO Channel 00/0 : 55[63000] -> 56[4000] [send] via NET/IB/0 comm 0x2b82f0000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 01/0 : 51[63000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b8468000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 01 : 24[4000] -> 25[26000] via P2P/IPC comm 0x2b09f8000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 00 : 52[4000] -> 53[26000] via P2P/IPC comm 0x2b8468000ab0 nRanks 96
e07r2n14:22278:23496 [3] NCCL INFO Channel 00/0 : 67[63000] -> 68[4000] [send] via NET/IB/0 comm 0x2b1038000ab0 nRanks 96
e07r2n15:28150:29329 [3] NCCL INFO Channel 00/0 : 71[63000] -> 72[4000] [send] via NET/IB/0 comm 0x2b3a44000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 01/0 : 63[63000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b696c000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 01/0 : 67[63000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b1e30000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 01/0 : 83[63000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5860000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 00 : 84[4000] -> 85[26000] via P2P/IPC comm 0x2b5860000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 00 : 64[4000] -> 65[26000] via P2P/IPC comm 0x2b696c000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 00 : 68[4000] -> 69[26000] via P2P/IPC comm 0x2b1e30000ab0 nRanks 96
e07r3n01:6046:7322 [0] NCCL INFO Channel 01/0 : 91[63000] -> 92[4000] [receive] via NET/IB/0 comm 0x2b0b68000ab0 nRanks 96
e07r3n01:6046:7322 [0] NCCL INFO Channel 00 : 92[4000] -> 93[26000] via P2P/IPC comm 0x2b0b68000ab0 nRanks 96
e07r2n12:16039:17014 [3] NCCL INFO Channel 00/0 : 59[63000] -> 60[4000] [send] via NET/IB/0 comm 0x2b3ec8000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 01 : 60[4000] -> 61[26000] via P2P/IPC comm 0x2ae9c8000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 01/0 : 55[63000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b831c000ab0 nRanks 96
e07r2n16:26998:28161 [3] NCCL INFO Channel 00/0 : 75[63000] -> 76[4000] [send] via NET/IB/0 comm 0x2b5c98000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 00 : 56[4000] -> 57[26000] via P2P/IPC comm 0x2b831c000ab0 nRanks 96
e07r1n19:30602:31679 [3] NCCL INFO Channel 00/0 : 7[63000] -> 8[4000] [send] via NET/IB/0 comm 0x2b2b9c000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 01 : 52[4000] -> 53[26000] via P2P/IPC comm 0x2b8468000ab0 nRanks 96
e07r2n02:4760:5858 [3] NCCL INFO Channel 00/0 : 19[63000] -> 20[4000] [send] via NET/IB/0 comm 0x2b72e8000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Channel 01/0 : 3[63000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b6e1c000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 01/0 : 71[63000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7c84000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Channel 00 : 4[4000] -> 5[26000] via P2P/IPC comm 0x2b6e1c000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 00 : 72[4000] -> 73[26000] via P2P/IPC comm 0x2b7c84000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 01/0 : 15[63000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b6f54000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 01/0 : 75[63000] -> 76[4000] [receive] via NET/IB/0 comm 0x2baf30000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 01 : 64[4000] -> 65[26000] via P2P/IPC comm 0x2b696c000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 01 : 68[4000] -> 69[26000] via P2P/IPC comm 0x2b1e30000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 01 : 84[4000] -> 85[26000] via P2P/IPC comm 0x2b5860000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 01/0 : 19[63000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3b20000ab0 nRanks 96
e07r1n18:15189:16547 [3] NCCL INFO Channel 00/0 : 3[63000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6d50000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 00 : 16[4000] -> 17[26000] via P2P/IPC comm 0x2b6f54000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 00 : 76[4000] -> 77[26000] via P2P/IPC comm 0x2baf30000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 00 : 20[4000] -> 21[26000] via P2P/IPC comm 0x2b3b20000ab0 nRanks 96
e07r2n01:629:1819 [3] NCCL INFO Channel 00/0 : 15[63000] -> 16[4000] [send] via NET/IB/0 comm 0x2ace88000ab0 nRanks 96
e07r2n00:23103:24289 [3] NCCL INFO Channel 00/0 : 11[63000] -> 12[4000] [send] via NET/IB/0 comm 0x2b8398000ab0 nRanks 96
e07r3n00:24493:25710 [3] NCCL INFO Channel 00/0 : 91[63000] -> 92[4000] [send] via NET/IB/0 comm 0x2b061c000ab0 nRanks 96
e07r3n01:6046:7322 [0] NCCL INFO Channel 01 : 92[4000] -> 93[26000] via P2P/IPC comm 0x2b0b68000ab0 nRanks 96
e07r1n18:15185:16349 [0] NCCL INFO Channel 01/0 : 95[63000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b2c88000ab0 nRanks 96
e07r1n18:15185:16349 [0] NCCL INFO Channel 00 : 0[4000] -> 1[26000] via P2P/IPC comm 0x2b2c88000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 01 : 16[4000] -> 17[26000] via P2P/IPC comm 0x2b6f54000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 01/0 : 11[63000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae748000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 01/0 : 7[63000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7be0000ab0 nRanks 96
e07r3n00:24490:25707 [0] NCCL INFO Channel 01/0 : 87[63000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b84fc000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 01 : 56[4000] -> 57[26000] via P2P/IPC comm 0x2b831c000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Channel 01 : 4[4000] -> 5[26000] via P2P/IPC comm 0x2b6e1c000ab0 nRanks 96
e07r2n18:25920:26909 [3] NCCL INFO Channel 00/0 : 83[63000] -> 84[4000] [send] via NET/IB/0 comm 0x2af5b8000ab0 nRanks 96
e07r2n13:31210:32209 [3] NCCL INFO Channel 00/0 : 63[63000] -> 64[4000] [send] via NET/IB/0 comm 0x2af840000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 00 : 12[4000] -> 13[26000] via P2P/IPC comm 0x2ae748000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 00 : 8[4000] -> 9[26000] via P2P/IPC comm 0x2b7be0000ab0 nRanks 96
e07r3n00:24490:25707 [0] NCCL INFO Channel 00 : 88[4000] -> 89[26000] via P2P/IPC comm 0x2b84fc000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 01/0 : 79[63000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b9534000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 01 : 76[4000] -> 77[26000] via P2P/IPC comm 0x2baf30000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 01 : 20[4000] -> 21[26000] via P2P/IPC comm 0x2b3b20000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 01 : 72[4000] -> 73[26000] via P2P/IPC comm 0x2b7c84000ab0 nRanks 96
e07r2n19:2111:3520 [3] NCCL INFO Channel 00/0 : 87[63000] -> 88[4000] [send] via NET/IB/0 comm 0x2b30c8000ab0 nRanks 96
e07r1n18:15185:16349 [0] NCCL INFO Channel 01 : 0[4000] -> 1[26000] via P2P/IPC comm 0x2b2c88000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 01 : 12[4000] -> 13[26000] via P2P/IPC comm 0x2ae748000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 01 : 8[4000] -> 9[26000] via P2P/IPC comm 0x2b7be0000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 00 : 80[4000] -> 81[26000] via P2P/IPC comm 0x2b9534000ab0 nRanks 96
e07r2n06:26609:27563 [1] NCCL INFO Connected all rings comm 0x2ae710000ab0 nRanks 96 busId 26000
e07r3n01:6049:7320 [3] NCCL INFO Channel 00/0 : 95[63000] -> 0[4000] [send] via NET/IB/0 comm 0x2b4e20000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 01 : 80[4000] -> 81[26000] via P2P/IPC comm 0x2b9534000ab0 nRanks 96
e07r3n00:24490:25707 [0] NCCL INFO Channel 01 : 88[4000] -> 89[26000] via P2P/IPC comm 0x2b84fc000ab0 nRanks 96
e07r2n05:907:2100 [1] NCCL INFO Connected all rings comm 0x2af6c0000ab0 nRanks 96 busId 26000
e07r2n17:31187:32200 [3] NCCL INFO Channel 00/0 : 79[63000] -> 80[4000] [send] via NET/IB/0 comm 0x2b1110000ab0 nRanks 96
e07r2n06:26609:27563 [1] NCCL INFO Channel 00/0 : 16[4000] -> 33[26000] [receive] via NET/IB/0 comm 0x2ae710000ab0 nRanks 96
e07r2n08:30378:31693 [1] NCCL INFO Connected all rings comm 0x2b2bac000ab0 nRanks 96 busId 26000
e07r2n05:907:2100 [1] NCCL INFO Channel 01/0 : 29[26000] -> 60[4000] [send] via NET/IB/0 comm 0x2af6c0000ab0 nRanks 96
e07r2n07:11608:12938 [1] NCCL INFO Connected all rings comm 0x2aeaa8000ab0 nRanks 96 busId 26000
e07r2n08:30378:31693 [1] NCCL INFO Channel 00/0 : 36[4000] -> 41[26000] [receive] via NET/IB/0 comm 0x2b2bac000ab0 nRanks 96
e07r2n07:11608:12938 [1] NCCL INFO Channel 01/0 : 37[26000] -> 40[4000] [send] via NET/IB/0 comm 0x2aeaa8000ab0 nRanks 96
e07r2n09:23024:24184 [1] NCCL INFO Connected all rings comm 0x2b39f0000ab0 nRanks 96 busId 26000
e07r2n10:29371:30326 [1] NCCL INFO Connected all rings comm 0x2ad4c4000ab0 nRanks 96 busId 26000
e07r2n04:1827:3073 [1] NCCL INFO Connected all rings comm 0x2abfb4000ab0 nRanks 96 busId 26000
e07r2n09:23024:24184 [1] NCCL INFO Channel 01/0 : 45[26000] -> 52[4000] [send] via NET/IB/0 comm 0x2b39f0000ab0 nRanks 96
e07r2n14:22276:23489 [1] NCCL INFO Connected all rings comm 0x2aee9c000ab0 nRanks 96 busId 26000
e07r2n10:29371:30326 [1] NCCL INFO Channel 00/0 : 40[4000] -> 49[26000] [receive] via NET/IB/0 comm 0x2ad4c4000ab0 nRanks 96
e07r2n11:16527:17685 [1] NCCL INFO Connected all rings comm 0x2ae98c000ab0 nRanks 96 busId 26000
e07r2n15:28147:29326 [1] NCCL INFO Connected all rings comm 0x2b19d0000ab0 nRanks 96 busId 26000
e07r2n04:1827:3073 [1] NCCL INFO Channel 00/0 : 20[4000] -> 25[26000] [receive] via NET/IB/0 comm 0x2abfb4000ab0 nRanks 96
e07r2n19:2109:3517 [1] NCCL INFO Connected all rings comm 0x2b2e10000ab0 nRanks 96 busId 26000
e07r2n13:31207:32206 [1] NCCL INFO Connected all rings comm 0x2aeb40000ab0 nRanks 96 busId 26000
e07r2n14:22276:23489 [1] NCCL INFO Channel 00/0 : 32[4000] -> 65[26000] [receive] via NET/IB/0 comm 0x2aee9c000ab0 nRanks 96
e07r1n19:30600:31684 [1] NCCL INFO Connected all rings comm 0x2b4630000ab0 nRanks 96 busId 26000
e07r2n11:16527:17685 [1] NCCL INFO Channel 01/0 : 53[26000] -> 56[4000] [send] via NET/IB/0 comm 0x2ae98c000ab0 nRanks 96
e07r2n12:16037:17013 [1] NCCL INFO Connected all rings comm 0x2ae0f4000ab0 nRanks 96 busId 26000
e07r3n01:6047:7323 [1] NCCL INFO Connected all rings comm 0x2adc2c000ab0 nRanks 96 busId 26000
e07r2n18:25918:26911 [1] NCCL INFO Connected all rings comm 0x2b094c000ab0 nRanks 96 busId 26000
e07r2n15:28147:29326 [1] NCCL INFO Channel 01/0 : 69[26000] -> 72[4000] [send] via NET/IB/0 comm 0x2b19d0000ab0 nRanks 96
e07r2n19:2109:3517 [1] NCCL INFO Channel 01/0 : 85[26000] -> 88[4000] [send] via NET/IB/0 comm 0x2b2e10000ab0 nRanks 96
e07r2n17:31185:32201 [1] NCCL INFO Connected all rings comm 0x2ac6ec000ab0 nRanks 96 busId 26000
e07r2n02:4758:5857 [1] NCCL INFO Connected all rings comm 0x2b8424000ab0 nRanks 96 busId 26000
e07r2n03:21864:23049 [1] NCCL INFO Connected all rings comm 0x2b3270000ab0 nRanks 96 busId 26000
e07r2n13:31207:32206 [1] NCCL INFO Channel 01/0 : 61[26000] -> 76[4000] [send] via NET/IB/0 comm 0x2aeb40000ab0 nRanks 96
e07r1n19:30600:31684 [1] NCCL INFO Channel 01/0 : 5[26000] -> 8[4000] [send] via NET/IB/0 comm 0x2b4630000ab0 nRanks 96
e07r2n01:627:1821 [1] NCCL INFO Connected all rings comm 0x2b75e4000ab0 nRanks 96 busId 26000
e07r2n16:26996:28158 [1] NCCL INFO Connected all rings comm 0x2ade8c000ab0 nRanks 96 busId 26000
e07r2n00:23101:24353 [1] NCCL INFO Connected all rings comm 0x2abe94000ab0 nRanks 96 busId 26000
e07r3n00:24491:25708 [1] NCCL INFO Connected all rings comm 0x2b9b5c000ab0 nRanks 96 busId 26000
e07r1n18:15187:16550 [1] NCCL INFO Connected all rings comm 0x2ac740000ab0 nRanks 96 busId 26000
e07r2n12:16037:17013 [1] NCCL INFO Channel 00/0 : 52[4000] -> 57[26000] [receive] via NET/IB/0 comm 0x2ae0f4000ab0 nRanks 96
e07r2n18:25918:26911 [1] NCCL INFO Channel 00/0 : 72[4000] -> 81[26000] [receive] via NET/IB/0 comm 0x2b094c000ab0 nRanks 96
e07r2n03:21864:23049 [1] NCCL INFO Channel 01/0 : 21[26000] -> 24[4000] [send] via NET/IB/0 comm 0x2b3270000ab0 nRanks 96
e07r2n17:31185:32201 [1] NCCL INFO Channel 01/0 : 77[26000] -> 84[4000] [send] via NET/IB/0 comm 0x2ac6ec000ab0 nRanks 96
e07r3n01:6047:7323 [1] NCCL INFO Channel 00 : 93[26000] -> 92[4000] via P2P/IPC comm 0x2adc2c000ab0 nRanks 96
e07r2n02:4758:5857 [1] NCCL INFO Channel 00/0 : 8[4000] -> 17[26000] [receive] via NET/IB/0 comm 0x2b8424000ab0 nRanks 96
e07r2n01:627:1821 [1] NCCL INFO Channel 01/0 : 13[26000] -> 20[4000] [send] via NET/IB/0 comm 0x2b75e4000ab0 nRanks 96
e07r2n00:23101:24353 [1] NCCL INFO Channel 00/0 : 4[4000] -> 9[26000] [receive] via NET/IB/0 comm 0x2abe94000ab0 nRanks 96
e07r2n16:26996:28158 [1] NCCL INFO Channel 00/0 : 68[4000] -> 73[26000] [receive] via NET/IB/0 comm 0x2ade8c000ab0 nRanks 96
e07r3n00:24491:25708 [1] NCCL INFO Channel 00/0 : 84[4000] -> 89[26000] [receive] via NET/IB/0 comm 0x2b9b5c000ab0 nRanks 96
e07r3n01:6047:7323 [1] NCCL INFO Channel 01 : 93[26000] -> 92[4000] via P2P/IPC comm 0x2adc2c000ab0 nRanks 96
e07r1n18:15187:16550 [1] NCCL INFO Channel 00 : 1[26000] -> 0[4000] via P2P/IPC comm 0x2ac740000ab0 nRanks 96
e07r1n18:15187:16550 [1] NCCL INFO Channel 01 : 1[26000] -> 0[4000] via P2P/IPC comm 0x2ac740000ab0 nRanks 96
e07r2n05:909:2102 [3] NCCL INFO Channel 01/0 : 31[63000] -> 32[4000] [send] via NET/IB/0 comm 0x2ac2e4000ab0 nRanks 96
e07r2n08:30380:31690 [3] NCCL INFO Channel 01/0 : 43[63000] -> 44[4000] [send] via NET/IB/0 comm 0x2b85a8000ab0 nRanks 96
e07r2n04:1829:3075 [3] NCCL INFO Channel 01/0 : 27[63000] -> 28[4000] [send] via NET/IB/0 comm 0x2b663c000ab0 nRanks 96
e07r2n07:11610:12943 [3] NCCL INFO Channel 01/0 : 39[63000] -> 40[4000] [send] via NET/IB/0 comm 0x2afd74000ab0 nRanks 96
e07r2n09:23026:24186 [3] NCCL INFO Channel 01/0 : 47[63000] -> 48[4000] [send] via NET/IB/0 comm 0x2b48e0000ab0 nRanks 96
e07r2n10:29373:30327 [3] NCCL INFO Channel 01/0 : 51[63000] -> 52[4000] [send] via NET/IB/0 comm 0x2b2b38000ab0 nRanks 96
e07r2n06:26611:27566 [3] NCCL INFO Channel 01/0 : 35[63000] -> 36[4000] [send] via NET/IB/0 comm 0x2b6078000ab0 nRanks 96
e07r2n03:21866:23047 [3] NCCL INFO Channel 01/0 : 23[63000] -> 24[4000] [send] via NET/IB/0 comm 0x2b6bd4000ab0 nRanks 96
e07r2n11:16529:17687 [3] NCCL INFO Channel 01/0 : 55[63000] -> 56[4000] [send] via NET/IB/0 comm 0x2b82f0000ab0 nRanks 96
e07r2n15:28150:29329 [3] NCCL INFO Channel 01/0 : 71[63000] -> 72[4000] [send] via NET/IB/0 comm 0x2b3a44000ab0 nRanks 96
e07r2n14:22278:23496 [3] NCCL INFO Channel 01/0 : 67[63000] -> 68[4000] [send] via NET/IB/0 comm 0x2b1038000ab0 nRanks 96
e07r2n12:16039:17014 [3] NCCL INFO Channel 01/0 : 59[63000] -> 60[4000] [send] via NET/IB/0 comm 0x2b3ec8000ab0 nRanks 96
e07r2n16:26998:28161 [3] NCCL INFO Channel 01/0 : 75[63000] -> 76[4000] [send] via NET/IB/0 comm 0x2b5c98000ab0 nRanks 96
e07r1n19:30602:31679 [3] NCCL INFO Channel 01/0 : 7[63000] -> 8[4000] [send] via NET/IB/0 comm 0x2b2b9c000ab0 nRanks 96
e07r2n02:4760:5858 [3] NCCL INFO Channel 01/0 : 19[63000] -> 20[4000] [send] via NET/IB/0 comm 0x2b72e8000ab0 nRanks 96
e07r2n05:908:2101 [2] NCCL INFO Connected all rings comm 0x2b0160000ab0 nRanks 96 busId 43000
e07r2n08:30379:31736 [2] NCCL INFO Connected all rings comm 0x2add50000ab0 nRanks 96 busId 43000
e07r2n01:629:1819 [3] NCCL INFO Channel 01/0 : 15[63000] -> 16[4000] [send] via NET/IB/0 comm 0x2ace88000ab0 nRanks 96
e07r2n04:1828:3077 [2] NCCL INFO Connected all rings comm 0x2b200c000ab0 nRanks 96 busId 43000
e07r1n18:15189:16547 [3] NCCL INFO Channel 01/0 : 3[63000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6d50000ab0 nRanks 96
e07r2n00:23103:24289 [3] NCCL INFO Channel 01/0 : 11[63000] -> 12[4000] [send] via NET/IB/0 comm 0x2b8398000ab0 nRanks 96
e07r3n00:24493:25710 [3] NCCL INFO Channel 01/0 : 91[63000] -> 92[4000] [send] via NET/IB/0 comm 0x2b061c000ab0 nRanks 96
e07r2n18:25920:26909 [3] NCCL INFO Channel 01/0 : 83[63000] -> 84[4000] [send] via NET/IB/0 comm 0x2af5b8000ab0 nRanks 96
e07r2n13:31210:32209 [3] NCCL INFO Channel 01/0 : 63[63000] -> 64[4000] [send] via NET/IB/0 comm 0x2af840000ab0 nRanks 96
e07r2n19:2111:3520 [3] NCCL INFO Channel 01/0 : 87[63000] -> 88[4000] [send] via NET/IB/0 comm 0x2b30c8000ab0 nRanks 96
e07r2n05:908:2101 [2] NCCL INFO Channel 00 : 30[43000] -> 29[26000] via P2P/IPC comm 0x2b0160000ab0 nRanks 96
e07r3n01:6049:7320 [3] NCCL INFO Channel 01/0 : 95[63000] -> 0[4000] [send] via NET/IB/0 comm 0x2b4e20000ab0 nRanks 96
e07r2n04:1828:3077 [2] NCCL INFO Channel 00 : 26[43000] -> 25[26000] via P2P/IPC comm 0x2b200c000ab0 nRanks 96
e07r2n07:11609:12944 [2] NCCL INFO Connected all rings comm 0x2aca80000ab0 nRanks 96 busId 43000
e07r2n08:30379:31736 [2] NCCL INFO Channel 00 : 42[43000] -> 41[26000] via P2P/IPC comm 0x2add50000ab0 nRanks 96
e07r2n09:23025:24185 [2] NCCL INFO Connected all rings comm 0x2b44d8000ab0 nRanks 96 busId 43000
e07r2n05:908:2101 [2] NCCL INFO Channel 01 : 30[43000] -> 29[26000] via P2P/IPC comm 0x2b0160000ab0 nRanks 96
e07r2n17:31187:32200 [3] NCCL INFO Channel 01/0 : 79[63000] -> 80[4000] [send] via NET/IB/0 comm 0x2b1110000ab0 nRanks 96
e07r2n04:1828:3077 [2] NCCL INFO Channel 01 : 26[43000] -> 25[26000] via P2P/IPC comm 0x2b200c000ab0 nRanks 96
e07r2n06:26610:27569 [2] NCCL INFO Connected all rings comm 0x2b9324000ab0 nRanks 96 busId 43000
e07r2n08:30379:31736 [2] NCCL INFO Channel 01 : 42[43000] -> 41[26000] via P2P/IPC comm 0x2add50000ab0 nRanks 96
e07r2n10:29372:30328 [2] NCCL INFO Connected all rings comm 0x2b4700000ab0 nRanks 96 busId 43000
e07r2n07:11609:12944 [2] NCCL INFO Channel 00 : 38[43000] -> 37[26000] via P2P/IPC comm 0x2aca80000ab0 nRanks 96
e07r2n09:23025:24185 [2] NCCL INFO Channel 00 : 46[43000] -> 45[26000] via P2P/IPC comm 0x2b44d8000ab0 nRanks 96
e07r2n03:21865:23050 [2] NCCL INFO Connected all rings comm 0x2b601c000ab0 nRanks 96 busId 43000
e07r2n06:26610:27569 [2] NCCL INFO Channel 00 : 34[43000] -> 33[26000] via P2P/IPC comm 0x2b9324000ab0 nRanks 96
e07r2n15:28148:29328 [2] NCCL INFO Connected all rings comm 0x2afa50000ab0 nRanks 96 busId 43000
e07r2n09:23025:24185 [2] NCCL INFO Channel 01 : 46[43000] -> 45[26000] via P2P/IPC comm 0x2b44d8000ab0 nRanks 96
e07r2n07:11609:12944 [2] NCCL INFO Channel 01 : 38[43000] -> 37[26000] via P2P/IPC comm 0x2aca80000ab0 nRanks 96
e07r2n11:16528:17686 [2] NCCL INFO Connected all rings comm 0x2b8a80000ab0 nRanks 96 busId 43000
e07r2n10:29372:30328 [2] NCCL INFO Channel 00 : 50[43000] -> 49[26000] via P2P/IPC comm 0x2b4700000ab0 nRanks 96
e07r2n16:26997:28160 [2] NCCL INFO Connected all rings comm 0x2b2168000ab0 nRanks 96 busId 43000
e07r2n06:26610:27569 [2] NCCL INFO Channel 01 : 34[43000] -> 33[26000] via P2P/IPC comm 0x2b9324000ab0 nRanks 96
e07r1n19:30601:31683 [2] NCCL INFO Connected all rings comm 0x2b9508000ab0 nRanks 96 busId 43000
e07r2n10:29372:30328 [2] NCCL INFO Channel 01 : 50[43000] -> 49[26000] via P2P/IPC comm 0x2b4700000ab0 nRanks 96
e07r2n03:21865:23050 [2] NCCL INFO Channel 00 : 22[43000] -> 21[26000] via P2P/IPC comm 0x2b601c000ab0 nRanks 96
e07r2n14:22277:23498 [2] NCCL INFO Connected all rings comm 0x2b736c000ab0 nRanks 96 busId 43000
e07r2n12:16038:17016 [2] NCCL INFO Connected all rings comm 0x2b93f8000ab0 nRanks 96 busId 43000
e07r2n15:28148:29328 [2] NCCL INFO Channel 00 : 70[43000] -> 69[26000] via P2P/IPC comm 0x2afa50000ab0 nRanks 96
e07r2n11:16528:17686 [2] NCCL INFO Channel 00 : 54[43000] -> 53[26000] via P2P/IPC comm 0x2b8a80000ab0 nRanks 96
e07r2n16:26997:28160 [2] NCCL INFO Channel 00 : 74[43000] -> 73[26000] via P2P/IPC comm 0x2b2168000ab0 nRanks 96
e07r2n03:21865:23050 [2] NCCL INFO Channel 01 : 22[43000] -> 21[26000] via P2P/IPC comm 0x2b601c000ab0 nRanks 96
e07r2n15:28148:29328 [2] NCCL INFO Channel 01 : 70[43000] -> 69[26000] via P2P/IPC comm 0x2afa50000ab0 nRanks 96
e07r2n00:23102:24358 [2] NCCL INFO Connected all rings comm 0x2acbf4000ab0 nRanks 96 busId 43000
e07r2n02:4759:5859 [2] NCCL INFO Connected all rings comm 0x2b32b8000ab0 nRanks 96 busId 43000
e07r2n11:16528:17686 [2] NCCL INFO Channel 01 : 54[43000] -> 53[26000] via P2P/IPC comm 0x2b8a80000ab0 nRanks 96
e07r1n18:15188:16551 [2] NCCL INFO Connected all rings comm 0x2b6a08000ab0 nRanks 96 busId 43000
e07r1n19:30601:31683 [2] NCCL INFO Channel 00 : 6[43000] -> 5[26000] via P2P/IPC comm 0x2b9508000ab0 nRanks 96
e07r2n01:628:1825 [2] NCCL INFO Connected all rings comm 0x2ae908000ab0 nRanks 96 busId 43000
e07r2n19:2110:3521 [2] NCCL INFO Connected all rings comm 0x2b9ce8000ab0 nRanks 96 busId 43000
e07r3n00:24492:25711 [2] NCCL INFO Connected all rings comm 0x2b96c4000ab0 nRanks 96 busId 43000
e07r2n13:31209:32208 [2] NCCL INFO Connected all rings comm 0x2ab7c4000ab0 nRanks 96 busId 43000
e07r2n18:25919:26914 [2] NCCL INFO Connected all rings comm 0x2aef38000ab0 nRanks 96 busId 43000
e07r2n16:26997:28160 [2] NCCL INFO Channel 01 : 74[43000] -> 73[26000] via P2P/IPC comm 0x2b2168000ab0 nRanks 96
e07r2n14:22277:23498 [2] NCCL INFO Channel 00 : 66[43000] -> 65[26000] via P2P/IPC comm 0x2b736c000ab0 nRanks 96
e07r1n19:30601:31683 [2] NCCL INFO Channel 01 : 6[43000] -> 5[26000] via P2P/IPC comm 0x2b9508000ab0 nRanks 96
e07r2n12:16038:17016 [2] NCCL INFO Channel 00 : 58[43000] -> 57[26000] via P2P/IPC comm 0x2b93f8000ab0 nRanks 96
e07r2n14:22277:23498 [2] NCCL INFO Channel 01 : 66[43000] -> 65[26000] via P2P/IPC comm 0x2b736c000ab0 nRanks 96
e07r2n12:16038:17016 [2] NCCL INFO Channel 01 : 58[43000] -> 57[26000] via P2P/IPC comm 0x2b93f8000ab0 nRanks 96
e07r2n02:4759:5859 [2] NCCL INFO Channel 00 : 18[43000] -> 17[26000] via P2P/IPC comm 0x2b32b8000ab0 nRanks 96
e07r2n19:2110:3521 [2] NCCL INFO Channel 00 : 86[43000] -> 85[26000] via P2P/IPC comm 0x2b9ce8000ab0 nRanks 96
e07r3n01:6048:7326 [2] NCCL INFO Connected all rings comm 0x2b3f2c000ab0 nRanks 96 busId 43000
e07r2n01:628:1825 [2] NCCL INFO Channel 00 : 14[43000] -> 13[26000] via P2P/IPC comm 0x2ae908000ab0 nRanks 96
e07r3n00:24492:25711 [2] NCCL INFO Channel 00 : 90[43000] -> 89[26000] via P2P/IPC comm 0x2b96c4000ab0 nRanks 96
e07r2n17:31186:32202 [2] NCCL INFO Connected all rings comm 0x2b2e7c000ab0 nRanks 96 busId 43000
e07r2n13:31209:32208 [2] NCCL INFO Channel 00 : 62[43000] -> 61[26000] via P2P/IPC comm 0x2ab7c4000ab0 nRanks 96
e07r2n00:23102:24358 [2] NCCL INFO Channel 00 : 10[43000] -> 9[26000] via P2P/IPC comm 0x2acbf4000ab0 nRanks 96
e07r2n18:25919:26914 [2] NCCL INFO Channel 00 : 82[43000] -> 81[26000] via P2P/IPC comm 0x2aef38000ab0 nRanks 96
e07r2n02:4759:5859 [2] NCCL INFO Channel 01 : 18[43000] -> 17[26000] via P2P/IPC comm 0x2b32b8000ab0 nRanks 96
e07r1n18:15188:16551 [2] NCCL INFO Channel 00 : 2[43000] -> 1[26000] via P2P/IPC comm 0x2b6a08000ab0 nRanks 96
e07r2n01:628:1825 [2] NCCL INFO Channel 01 : 14[43000] -> 13[26000] via P2P/IPC comm 0x2ae908000ab0 nRanks 96
e07r3n00:24492:25711 [2] NCCL INFO Channel 01 : 90[43000] -> 89[26000] via P2P/IPC comm 0x2b96c4000ab0 nRanks 96
e07r2n13:31209:32208 [2] NCCL INFO Channel 01 : 62[43000] -> 61[26000] via P2P/IPC comm 0x2ab7c4000ab0 nRanks 96
e07r2n00:23102:24358 [2] NCCL INFO Channel 01 : 10[43000] -> 9[26000] via P2P/IPC comm 0x2acbf4000ab0 nRanks 96
e07r2n19:2110:3521 [2] NCCL INFO Channel 01 : 86[43000] -> 85[26000] via P2P/IPC comm 0x2b9ce8000ab0 nRanks 96
e07r1n18:15188:16551 [2] NCCL INFO Channel 01 : 2[43000] -> 1[26000] via P2P/IPC comm 0x2b6a08000ab0 nRanks 96
e07r2n18:25919:26914 [2] NCCL INFO Channel 01 : 82[43000] -> 81[26000] via P2P/IPC comm 0x2aef38000ab0 nRanks 96
e07r3n01:6048:7326 [2] NCCL INFO Channel 00 : 94[43000] -> 93[26000] via P2P/IPC comm 0x2b3f2c000ab0 nRanks 96
e07r2n17:31186:32202 [2] NCCL INFO Channel 00 : 78[43000] -> 77[26000] via P2P/IPC comm 0x2b2e7c000ab0 nRanks 96
e07r3n01:6048:7326 [2] NCCL INFO Channel 01 : 94[43000] -> 93[26000] via P2P/IPC comm 0x2b3f2c000ab0 nRanks 96
e07r2n17:31186:32202 [2] NCCL INFO Channel 01 : 78[43000] -> 77[26000] via P2P/IPC comm 0x2b2e7c000ab0 nRanks 96
e07r2n05:909:2102 [3] NCCL INFO Connected all rings comm 0x2ac2e4000ab0 nRanks 96 busId 63000
e07r2n05:909:2102 [3] NCCL INFO Channel 00 : 31[63000] -> 30[43000] via P2P/IPC comm 0x2ac2e4000ab0 nRanks 96
e07r2n05:909:2102 [3] NCCL INFO Channel 01 : 31[63000] -> 30[43000] via P2P/IPC comm 0x2ac2e4000ab0 nRanks 96
e07r2n09:23026:24186 [3] NCCL INFO Connected all rings comm 0x2b48e0000ab0 nRanks 96 busId 63000
e07r2n09:23026:24186 [3] NCCL INFO Channel 00 : 47[63000] -> 46[43000] via P2P/IPC comm 0x2b48e0000ab0 nRanks 96
e07r2n09:23026:24186 [3] NCCL INFO Channel 01 : 47[63000] -> 46[43000] via P2P/IPC comm 0x2b48e0000ab0 nRanks 96
e07r2n08:30380:31690 [3] NCCL INFO Connected all rings comm 0x2b85a8000ab0 nRanks 96 busId 63000
e07r2n08:30380:31690 [3] NCCL INFO Channel 00 : 43[63000] -> 42[43000] via P2P/IPC comm 0x2b85a8000ab0 nRanks 96
e07r2n08:30380:31690 [3] NCCL INFO Channel 01 : 43[63000] -> 42[43000] via P2P/IPC comm 0x2b85a8000ab0 nRanks 96
e07r2n03:21866:23047 [3] NCCL INFO Connected all rings comm 0x2b6bd4000ab0 nRanks 96 busId 63000
e07r2n03:21866:23047 [3] NCCL INFO Channel 00 : 23[63000] -> 22[43000] via P2P/IPC comm 0x2b6bd4000ab0 nRanks 96
e07r2n05:909:2102 [3] NCCL INFO Connected all trees comm 0x2ac2e4000ab0 nRanks 96 busId 63000
e07r2n05:909:2102 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:909:2102 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n10:29373:30327 [3] NCCL INFO Connected all rings comm 0x2b2b38000ab0 nRanks 96 busId 63000
e07r2n06:26611:27566 [3] NCCL INFO Connected all rings comm 0x2b6078000ab0 nRanks 96 busId 63000
e07r2n10:29373:30327 [3] NCCL INFO Channel 00 : 51[63000] -> 50[43000] via P2P/IPC comm 0x2b2b38000ab0 nRanks 96
e07r2n06:26611:27566 [3] NCCL INFO Channel 00 : 35[63000] -> 34[43000] via P2P/IPC comm 0x2b6078000ab0 nRanks 96
e07r2n07:11610:12943 [3] NCCL INFO Connected all rings comm 0x2afd74000ab0 nRanks 96 busId 63000
e07r2n07:11610:12943 [3] NCCL INFO Channel 00 : 39[63000] -> 38[43000] via P2P/IPC comm 0x2afd74000ab0 nRanks 96
e07r2n14:22278:23496 [3] NCCL INFO Connected all rings comm 0x2b1038000ab0 nRanks 96 busId 63000
e07r2n14:22278:23496 [3] NCCL INFO Channel 00 : 67[63000] -> 66[43000] via P2P/IPC comm 0x2b1038000ab0 nRanks 96
e07r2n03:21866:23047 [3] NCCL INFO Channel 01 : 23[63000] -> 22[43000] via P2P/IPC comm 0x2b6bd4000ab0 nRanks 96
e07r2n11:16529:17687 [3] NCCL INFO Connected all rings comm 0x2b82f0000ab0 nRanks 96 busId 63000
e07r2n11:16529:17687 [3] NCCL INFO Channel 00 : 55[63000] -> 54[43000] via P2P/IPC comm 0x2b82f0000ab0 nRanks 96
e07r2n10:29373:30327 [3] NCCL INFO Channel 01 : 51[63000] -> 50[43000] via P2P/IPC comm 0x2b2b38000ab0 nRanks 96
e07r2n06:26611:27566 [3] NCCL INFO Channel 01 : 35[63000] -> 34[43000] via P2P/IPC comm 0x2b6078000ab0 nRanks 96
e07r2n01:629:1819 [3] NCCL INFO Connected all rings comm 0x2ace88000ab0 nRanks 96 busId 63000
e07r2n01:629:1819 [3] NCCL INFO Channel 00 : 15[63000] -> 14[43000] via P2P/IPC comm 0x2ace88000ab0 nRanks 96
e07r2n07:11610:12943 [3] NCCL INFO Channel 01 : 39[63000] -> 38[43000] via P2P/IPC comm 0x2afd74000ab0 nRanks 96
e07r2n14:22278:23496 [3] NCCL INFO Channel 01 : 67[63000] -> 66[43000] via P2P/IPC comm 0x2b1038000ab0 nRanks 96
e07r2n11:16529:17687 [3] NCCL INFO Channel 01 : 55[63000] -> 54[43000] via P2P/IPC comm 0x2b82f0000ab0 nRanks 96
e07r2n15:28150:29329 [3] NCCL INFO Connected all rings comm 0x2b3a44000ab0 nRanks 96 busId 63000
e07r2n15:28150:29329 [3] NCCL INFO Channel 00 : 71[63000] -> 70[43000] via P2P/IPC comm 0x2b3a44000ab0 nRanks 96
e07r3n00:24493:25710 [3] NCCL INFO Connected all rings comm 0x2b061c000ab0 nRanks 96 busId 63000
e07r2n01:629:1819 [3] NCCL INFO Channel 01 : 15[63000] -> 14[43000] via P2P/IPC comm 0x2ace88000ab0 nRanks 96
e07r3n00:24493:25710 [3] NCCL INFO Channel 00 : 91[63000] -> 90[43000] via P2P/IPC comm 0x2b061c000ab0 nRanks 96
e07r2n09:23026:24186 [3] NCCL INFO Connected all trees comm 0x2b48e0000ab0 nRanks 96 busId 63000
e07r2n09:23026:24186 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n09:23026:24186 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n19:2111:3520 [3] NCCL INFO Connected all rings comm 0x2b30c8000ab0 nRanks 96 busId 63000
e07r2n16:26998:28161 [3] NCCL INFO Connected all rings comm 0x2b5c98000ab0 nRanks 96 busId 63000
e07r2n02:4760:5858 [3] NCCL INFO Connected all rings comm 0x2b72e8000ab0 nRanks 96 busId 63000
e07r1n18:15189:16547 [3] NCCL INFO Connected all rings comm 0x2b6d50000ab0 nRanks 96 busId 63000
e07r1n18:15189:16547 [3] NCCL INFO Channel 00 : 3[63000] -> 2[43000] via P2P/IPC comm 0x2b6d50000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Connected all rings comm 0x2af760000ab0 nRanks 96 busId 4000
e07r2n19:2111:3520 [3] NCCL INFO Channel 00 : 87[63000] -> 86[43000] via P2P/IPC comm 0x2b30c8000ab0 nRanks 96
e07r2n16:26998:28161 [3] NCCL INFO Channel 00 : 75[63000] -> 74[43000] via P2P/IPC comm 0x2b5c98000ab0 nRanks 96
e07r2n02:4760:5858 [3] NCCL INFO Channel 00 : 19[63000] -> 18[43000] via P2P/IPC comm 0x2b72e8000ab0 nRanks 96
e07r2n12:16039:17014 [3] NCCL INFO Connected all rings comm 0x2b3ec8000ab0 nRanks 96 busId 63000
e07r2n12:16039:17014 [3] NCCL INFO Channel 00 : 59[63000] -> 58[43000] via P2P/IPC comm 0x2b3ec8000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Connected all rings comm 0x2b24f0000ab0 nRanks 96 busId 4000
e07r2n06:26608:27568 [0] NCCL INFO Connected all rings comm 0x2b2990000ab0 nRanks 96 busId 4000
e07r2n00:23103:24289 [3] NCCL INFO Connected all rings comm 0x2b8398000ab0 nRanks 96 busId 63000
e07r3n00:24493:25710 [3] NCCL INFO Channel 01 : 91[63000] -> 90[43000] via P2P/IPC comm 0x2b061c000ab0 nRanks 96
e07r2n15:28150:29329 [3] NCCL INFO Channel 01 : 71[63000] -> 70[43000] via P2P/IPC comm 0x2b3a44000ab0 nRanks 96
e07r2n12:16039:17014 [3] NCCL INFO Channel 01 : 59[63000] -> 58[43000] via P2P/IPC comm 0x2b3ec8000ab0 nRanks 96
e07r2n00:23103:24289 [3] NCCL INFO Channel 00 : 11[63000] -> 10[43000] via P2P/IPC comm 0x2b8398000ab0 nRanks 96
e07r2n18:25920:26909 [3] NCCL INFO Connected all rings comm 0x2af5b8000ab0 nRanks 96 busId 63000
e07r1n18:15189:16547 [3] NCCL INFO Channel 01 : 3[63000] -> 2[43000] via P2P/IPC comm 0x2b6d50000ab0 nRanks 96
e07r2n18:25920:26909 [3] NCCL INFO Channel 00 : 83[63000] -> 82[43000] via P2P/IPC comm 0x2af5b8000ab0 nRanks 96
e07r2n16:26998:28161 [3] NCCL INFO Channel 01 : 75[63000] -> 74[43000] via P2P/IPC comm 0x2b5c98000ab0 nRanks 96
e07r3n01:6049:7320 [3] NCCL INFO Connected all rings comm 0x2b4e20000ab0 nRanks 96 busId 63000
e07r3n01:6049:7320 [3] NCCL INFO Channel 00 : 95[63000] -> 94[43000] via P2P/IPC comm 0x2b4e20000ab0 nRanks 96
e07r2n00:23103:24289 [3] NCCL INFO Channel 01 : 11[63000] -> 10[43000] via P2P/IPC comm 0x2b8398000ab0 nRanks 96
e07r2n19:2111:3520 [3] NCCL INFO Channel 01 : 87[63000] -> 86[43000] via P2P/IPC comm 0x2b30c8000ab0 nRanks 96
e07r2n02:4760:5858 [3] NCCL INFO Channel 01 : 19[63000] -> 18[43000] via P2P/IPC comm 0x2b72e8000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Connected all rings comm 0x2b8468000ab0 nRanks 96 busId 4000
e07r2n18:25920:26909 [3] NCCL INFO Channel 01 : 83[63000] -> 82[43000] via P2P/IPC comm 0x2af5b8000ab0 nRanks 96
e07r3n01:6049:7320 [3] NCCL INFO Channel 01 : 95[63000] -> 94[43000] via P2P/IPC comm 0x2b4e20000ab0 nRanks 96
e07r2n08:30380:31690 [3] NCCL INFO Connected all trees comm 0x2b85a8000ab0 nRanks 96 busId 63000
e07r2n08:30380:31690 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n08:30380:31690 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n08:30377:31697 [0] NCCL INFO Connected all rings comm 0x2b4008000ab0 nRanks 96 busId 4000
e07r2n15:28145:29327 [0] NCCL INFO Connected all rings comm 0x2b1e30000ab0 nRanks 96 busId 4000
e07r2n03:21866:23047 [3] NCCL INFO Connected all trees comm 0x2b6bd4000ab0 nRanks 96 busId 63000
e07r2n03:21866:23047 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31187:32200 [3] NCCL INFO Connected all rings comm 0x2b1110000ab0 nRanks 96 busId 63000
e07r2n03:21866:23047 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n10:29373:30327 [3] NCCL INFO Connected all trees comm 0x2b2b38000ab0 nRanks 96 busId 63000
e07r2n12:16035:17011 [0] NCCL INFO Connected all rings comm 0x2b831c000ab0 nRanks 96 busId 4000
e07r2n17:31187:32200 [3] NCCL INFO Channel 00 : 79[63000] -> 78[43000] via P2P/IPC comm 0x2b1110000ab0 nRanks 96
e07r1n19:30602:31679 [3] NCCL INFO Connected all rings comm 0x2b2b9c000ab0 nRanks 96 busId 63000
e07r2n06:26611:27566 [3] NCCL INFO Connected all trees comm 0x2b6078000ab0 nRanks 96 busId 63000
e07r1n19:30602:31679 [3] NCCL INFO Channel 00 : 7[63000] -> 6[43000] via P2P/IPC comm 0x2b2b9c000ab0 nRanks 96
e07r2n06:26611:27566 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n10:29373:30327 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11610:12943 [3] NCCL INFO Connected all trees comm 0x2afd74000ab0 nRanks 96 busId 63000
e07r2n06:26611:27566 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n10:29373:30327 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n07:11607:12941 [0] NCCL INFO Connected all rings comm 0x2ac774000ab0 nRanks 96 busId 4000
e07r2n07:11610:12943 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11610:12943 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22278:23496 [3] NCCL INFO Connected all trees comm 0x2b1038000ab0 nRanks 96 busId 63000
e07r2n14:22278:23496 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16529:17687 [3] NCCL INFO Connected all trees comm 0x2b82f0000ab0 nRanks 96 busId 63000
e07r2n14:22278:23496 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16529:17687 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16529:17687 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n19:30602:31679 [3] NCCL INFO Channel 01 : 7[63000] -> 6[43000] via P2P/IPC comm 0x2b2b9c000ab0 nRanks 96
e07r2n01:629:1819 [3] NCCL INFO Connected all trees comm 0x2ace88000ab0 nRanks 96 busId 63000
e07r1n19:30599:31677 [0] NCCL INFO Connected all rings comm 0x2b6e1c000ab0 nRanks 96 busId 4000
e07r2n01:629:1819 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4756:5860 [0] NCCL INFO Connected all rings comm 0x2b6f54000ab0 nRanks 96 busId 4000
e07r2n01:629:1819 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n17:31187:32200 [3] NCCL INFO Channel 01 : 79[63000] -> 78[43000] via P2P/IPC comm 0x2b1110000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Connected all rings comm 0x2b09f8000ab0 nRanks 96 busId 4000
e07r2n01:626:1823 [0] NCCL INFO Connected all rings comm 0x2ae748000ab0 nRanks 96 busId 4000
e07r2n03:21863:23045 [0] NCCL INFO Connected all rings comm 0x2b3b20000ab0 nRanks 96 busId 4000
e07r3n00:24490:25707 [0] NCCL INFO Connected all rings comm 0x2b84fc000ab0 nRanks 96 busId 4000
e07r2n15:28150:29329 [3] NCCL INFO Connected all trees comm 0x2b3a44000ab0 nRanks 96 busId 63000
e07r3n00:24493:25710 [3] NCCL INFO Connected all trees comm 0x2b061c000ab0 nRanks 96 busId 63000
e07r3n00:24493:25710 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28150:29329 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n00:24493:25710 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28150:29329 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26998:28161 [3] NCCL INFO Connected all trees comm 0x2b5c98000ab0 nRanks 96 busId 63000
e07r2n12:16039:17014 [3] NCCL INFO Connected all trees comm 0x2b3ec8000ab0 nRanks 96 busId 63000
e07r2n12:16039:17014 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n12:16039:17014 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26998:28161 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n18:15189:16547 [3] NCCL INFO Connected all trees comm 0x2b6d50000ab0 nRanks 96 busId 63000
e07r1n18:15189:16547 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n16:26995:28159 [0] NCCL INFO Connected all rings comm 0x2b7c84000ab0 nRanks 96 busId 4000
e07r2n19:2111:3520 [3] NCCL INFO Connected all trees comm 0x2b30c8000ab0 nRanks 96 busId 63000
e07r2n00:23103:24289 [3] NCCL INFO Connected all trees comm 0x2b8398000ab0 nRanks 96 busId 63000
e07r1n18:15189:16547 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26998:28161 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n19:2111:3520 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23103:24289 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n19:2111:3520 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23103:24289 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n18:25920:26909 [3] NCCL INFO Connected all trees comm 0x2af5b8000ab0 nRanks 96 busId 63000
e07r1n18:15188:16551 [2] NCCL INFO Connected all trees comm 0x2b6a08000ab0 nRanks 96 busId 43000
e07r1n18:15188:16551 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n18:15188:16551 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n18:25920:26909 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n18:25920:26909 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n01:6049:7320 [3] NCCL INFO Connected all trees comm 0x2b4e20000ab0 nRanks 96 busId 63000
e07r2n17:31184:32198 [0] NCCL INFO Connected all rings comm 0x2baf30000ab0 nRanks 96 busId 4000
e07r3n01:6049:7320 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6049:7320 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n02:4760:5858 [3] NCCL INFO Connected all trees comm 0x2b72e8000ab0 nRanks 96 busId 63000
e07r2n02:4760:5858 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4760:5858 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n01:6048:7326 [2] NCCL INFO Connected all trees comm 0x2b3f2c000ab0 nRanks 96 busId 43000
e07r2n05:906:2099 [0] NCCL INFO Connected all rings comm 0x2ba3ac000ab0 nRanks 96 busId 4000
e07r3n01:6048:7326 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6048:7326 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n19:2107:3519 [0] NCCL INFO Connected all rings comm 0x2b5860000ab0 nRanks 96 busId 4000
e07r1n19:30602:31679 [3] NCCL INFO Connected all trees comm 0x2b2b9c000ab0 nRanks 96 busId 63000
e07r1n18:15185:16349 [0] NCCL INFO Connected all rings comm 0x2b2c88000ab0 nRanks 96 busId 4000
e07r1n19:30602:31679 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n19:30602:31679 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n17:31187:32200 [3] NCCL INFO Connected all trees comm 0x2b1110000ab0 nRanks 96 busId 63000
e07r3n01:6046:7322 [0] NCCL INFO Connected all rings comm 0x2b0b68000ab0 nRanks 96 busId 4000
e07r2n17:31187:32200 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31187:32200 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n18:25917:26912 [0] NCCL INFO Connected all rings comm 0x2b9534000ab0 nRanks 96 busId 4000
e07r2n13:31210:32209 [3] NCCL INFO Connected all rings comm 0x2af840000ab0 nRanks 96 busId 63000
e07r2n13:31210:32209 [3] NCCL INFO Channel 00 : 63[63000] -> 62[43000] via P2P/IPC comm 0x2af840000ab0 nRanks 96
e07r2n13:31210:32209 [3] NCCL INFO Channel 01 : 63[63000] -> 62[43000] via P2P/IPC comm 0x2af840000ab0 nRanks 96
e07r2n04:1829:3075 [3] NCCL INFO Connected all rings comm 0x2b663c000ab0 nRanks 96 busId 63000
e07r2n04:1829:3075 [3] NCCL INFO Channel 00 : 27[63000] -> 26[43000] via P2P/IPC comm 0x2b663c000ab0 nRanks 96
e07r2n04:1829:3075 [3] NCCL INFO Channel 01 : 27[63000] -> 26[43000] via P2P/IPC comm 0x2b663c000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Connected all rings comm 0x2b696c000ab0 nRanks 96 busId 4000
e07r2n13:31210:32209 [3] NCCL INFO Connected all trees comm 0x2af840000ab0 nRanks 96 busId 63000
e07r2n13:31210:32209 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31210:32209 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n04:1829:3075 [3] NCCL INFO Connected all trees comm 0x2b663c000ab0 nRanks 96 busId 63000
e07r2n04:1829:3075 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n04:1829:3075 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n13:31206:32207 [0] NCCL INFO Connected all rings comm 0x2ae9c8000ab0 nRanks 96 busId 4000
e07r2n00:23099:24278 [0] NCCL INFO Connected all rings comm 0x2b7be0000ab0 nRanks 96 busId 4000
e07r2n09:23023:24187 [0] NCCL INFO Channel 00/0 : 40[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af760000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 01/0 : 48[4000] -> 52[4000] [send] via NET/IB/0 comm 0x2b24f0000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 01/0 : 32[4000] -> 36[4000] [send] via NET/IB/0 comm 0x2b2990000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 01/0 : 48[4000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b8468000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 01/0 : 64[4000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b1e30000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 01/0 : 53[26000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b831c000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 01/0 : 16[4000] -> 20[4000] [send] via NET/IB/0 comm 0x2b6f54000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Channel 01/0 : 0[4000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b6e1c000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 01/0 : 37[26000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4008000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 01/0 : 21[26000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b09f8000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 00/0 : 8[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae748000ab0 nRanks 96
e07r3n00:24490:25707 [0] NCCL INFO Channel 01/0 : 85[26000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b84fc000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 01/0 : 32[4000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac774000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 01/0 : 69[26000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7c84000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 00/0 : 72[4000] -> 76[4000] [receive] via NET/IB/0 comm 0x2baf30000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 01/0 : 16[4000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3b20000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 00/0 : 24[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba3ac000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 01/0 : 80[4000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5860000ab0 nRanks 96
e07r1n18:15185:16349 [0] NCCL INFO Channel 01/0 : 0[4000] -> 4[4000] [send] via NET/IB/0 comm 0x2b2c88000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 01/0 : 80[4000] -> 84[4000] [send] via NET/IB/0 comm 0x2b9534000ab0 nRanks 96
e07r3n01:6046:7322 [0] NCCL INFO Channel 00/0 : 88[4000] -> 92[4000] [receive] via NET/IB/0 comm 0x2b0b68000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 01/0 : 64[4000] -> 68[4000] [send] via NET/IB/0 comm 0x2b696c000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 00/0 : 56[4000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ae9c8000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 01/0 : 5[26000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7be0000ab0 nRanks 96
e07r2n11:16527:17685 [1] NCCL INFO Channel 01/0 : 56[4000] -> 53[26000] [receive] via NET/IB/0 comm 0x2ae98c000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 00/0 : 48[4000] -> 56[4000] [send] via NET/IB/0 comm 0x2b24f0000ab0 nRanks 96
e07r2n19:2109:3517 [1] NCCL INFO Channel 01/0 : 88[4000] -> 85[26000] [receive] via NET/IB/0 comm 0x2b2e10000ab0 nRanks 96
e07r2n07:11608:12938 [1] NCCL INFO Channel 01/0 : 40[4000] -> 37[26000] [receive] via NET/IB/0 comm 0x2aeaa8000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 00/0 : 32[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2b2990000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 00/0 : 16[4000] -> 24[4000] [send] via NET/IB/0 comm 0x2b6f54000ab0 nRanks 96
e07r2n15:28147:29326 [1] NCCL INFO Channel 01/0 : 72[4000] -> 69[26000] [receive] via NET/IB/0 comm 0x2b19d0000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 00/0 : 52[4000] -> 57[26000] [send] via NET/IB/0 comm 0x2b8468000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 00/0 : 36[4000] -> 41[26000] [send] via NET/IB/0 comm 0x2ac774000ab0 nRanks 96
e07r2n03:21864:23049 [1] NCCL INFO Channel 01/0 : 24[4000] -> 21[26000] [receive] via NET/IB/0 comm 0x2b3270000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 00/0 : 80[4000] -> 88[4000] [send] via NET/IB/0 comm 0x2b9534000ab0 nRanks 96
e07r1n18:15185:16349 [0] NCCL INFO Channel 00/0 : 64[4000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b2c88000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 00/0 : 24[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2b09f8000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 00/0 : 20[4000] -> 25[26000] [send] via NET/IB/0 comm 0x2b3b20000ab0 nRanks 96
e07r3n00:24490:25707 [0] NCCL INFO Channel 00/0 : 88[4000] -> 92[4000] [send] via NET/IB/0 comm 0x2b84fc000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 00/0 : 56[4000] -> 60[4000] [send] via NET/IB/0 comm 0x2b831c000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 00/0 : 40[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2b4008000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 00/0 : 84[4000] -> 89[26000] [send] via NET/IB/0 comm 0x2b5860000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 00/0 : 64[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b696c000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 00/0 : 72[4000] -> 76[4000] [send] via NET/IB/0 comm 0x2b7c84000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Channel 00/0 : 4[4000] -> 9[26000] [send] via NET/IB/0 comm 0x2b6e1c000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 00/0 : 68[4000] -> 73[26000] [send] via NET/IB/0 comm 0x2b1e30000ab0 nRanks 96
e07r1n19:30600:31684 [1] NCCL INFO Channel 01/0 : 8[4000] -> 5[26000] [receive] via NET/IB/0 comm 0x2b4630000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 00/0 : 8[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2b7be0000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 01/0 : 36[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2ac774000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 01/0 : 45[26000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b8468000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 00/0 : 16[4000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b09f8000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 01/0 : 13[26000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3b20000ab0 nRanks 96
e07r3n00:24490:25707 [0] NCCL INFO Channel 00/0 : 80[4000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b84fc000ab0 nRanks 96
e07r2n12:16037:17013 [1] NCCL INFO Channel 00/0 : 57[26000] -> 52[4000] [send] via NET/IB/0 comm 0x2ae0f4000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 00/0 : 48[4000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b831c000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 00/0 : 40[4000] -> 49[26000] [send] via NET/IB/0 comm 0x2b4008000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 01/0 : 77[26000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5860000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 00/0 : 72[4000] -> 81[26000] [send] via NET/IB/0 comm 0x2b7c84000ab0 nRanks 96
e07r2n08:30378:31693 [1] NCCL INFO Channel 00/0 : 41[26000] -> 36[4000] [send] via NET/IB/0 comm 0x2b2bac000ab0 nRanks 96
e07r2n04:1827:3073 [1] NCCL INFO Channel 00/0 : 25[26000] -> 20[4000] [send] via NET/IB/0 comm 0x2abfb4000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Channel 01/0 : 4[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2b6e1c000ab0 nRanks 96
e07r3n00:24491:25708 [1] NCCL INFO Channel 00/0 : 89[26000] -> 84[4000] [send] via NET/IB/0 comm 0x2b9b5c000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 01/0 : 12[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba3ac000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 01/0 : 68[4000] -> 76[4000] [send] via NET/IB/0 comm 0x2b1e30000ab0 nRanks 96
e07r3n01:6046:7322 [0] NCCL INFO Channel 01/0 : 92[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2b0b68000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 01/0 : 36[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af760000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 01/0 : 44[4000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ae9c8000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 01/0 : 68[4000] -> 76[4000] [receive] via NET/IB/0 comm 0x2baf30000ab0 nRanks 96
e07r2n16:26996:28158 [1] NCCL INFO Channel 00/0 : 73[26000] -> 68[4000] [send] via NET/IB/0 comm 0x2ade8c000ab0 nRanks 96
e07r2n00:23101:24353 [1] NCCL INFO Channel 00/0 : 9[26000] -> 4[4000] [send] via NET/IB/0 comm 0x2abe94000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 00/0 : 8[4000] -> 17[26000] [send] via NET/IB/0 comm 0x2b7be0000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 01/0 : 4[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae748000ab0 nRanks 96
e07r2n09:23024:24184 [1] NCCL INFO Channel 01/0 : 52[4000] -> 45[26000] [receive] via NET/IB/0 comm 0x2b39f0000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 00/0 : 64[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b9534000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 00/0 : 16[4000] -> 33[26000] [send] via NET/IB/0 comm 0x2b6f54000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 00/0 : 49[26000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4008000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 01/0 : 52[4000] -> 45[26000] [send] via NET/IB/0 comm 0x2b8468000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 00/0 : 32[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b24f0000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 00/0 : 81[26000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7c84000ab0 nRanks 96
e07r2n17:31185:32201 [1] NCCL INFO Channel 01/0 : 84[4000] -> 77[26000] [receive] via NET/IB/0 comm 0x2ac6ec000ab0 nRanks 96
e07r2n01:627:1821 [1] NCCL INFO Channel 01/0 : 20[4000] -> 13[26000] [receive] via NET/IB/0 comm 0x2b75e4000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 01/0 : 20[4000] -> 13[26000] [send] via NET/IB/0 comm 0x2b3b20000ab0 nRanks 96
e07r2n10:29371:30326 [1] NCCL INFO Channel 00/0 : 49[26000] -> 40[4000] [send] via NET/IB/0 comm 0x2ad4c4000ab0 nRanks 96
e07r3n00:24490:25707 [0] NCCL INFO Channel 00/0 : 88[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b84fc000ab0 nRanks 96
e07r2n18:25918:26911 [1] NCCL INFO Channel 00/0 : 81[26000] -> 72[4000] [send] via NET/IB/0 comm 0x2b094c000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 00/0 : 56[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2b831c000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 00/0 : 24[4000] -> 16[4000] [send] via NET/IB/0 comm 0x2b09f8000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 01/0 : 44[4000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac774000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 01/0 : 84[4000] -> 77[26000] [send] via NET/IB/0 comm 0x2b5860000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 01/0 : 76[4000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b1e30000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 01/0 : 61[26000] -> 76[4000] [receive] via NET/IB/0 comm 0x2baf30000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 01/0 : 44[4000] -> 60[4000] [send] via NET/IB/0 comm 0x2af760000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 00/0 : 17[26000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7be0000ab0 nRanks 96
e07r2n02:4758:5857 [1] NCCL INFO Channel 00/0 : 17[26000] -> 8[4000] [send] via NET/IB/0 comm 0x2b8424000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Channel 01/0 : 12[4000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b6e1c000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 01/0 : 12[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2ae748000ab0 nRanks 96
e07r2n18:25918:26911 [1] NCCL INFO Channel 00 : 81[26000] -> 80[4000] via P2P/IPC comm 0x2b094c000ab0 nRanks 96
e07r2n18:25918:26911 [1] NCCL INFO Channel 01 : 81[26000] -> 80[4000] via P2P/IPC comm 0x2b094c000ab0 nRanks 96
e07r2n09:23024:24184 [1] NCCL INFO Channel 00 : 45[26000] -> 44[4000] via P2P/IPC comm 0x2b39f0000ab0 nRanks 96
e07r2n09:23024:24184 [1] NCCL INFO Channel 01 : 45[26000] -> 44[4000] via P2P/IPC comm 0x2b39f0000ab0 nRanks 96
e07r2n10:29371:30326 [1] NCCL INFO Channel 00 : 49[26000] -> 48[4000] via P2P/IPC comm 0x2ad4c4000ab0 nRanks 96
e07r2n10:29371:30326 [1] NCCL INFO Channel 01 : 49[26000] -> 48[4000] via P2P/IPC comm 0x2ad4c4000ab0 nRanks 96
e07r2n18:25919:26914 [2] NCCL INFO Connected all trees comm 0x2aef38000ab0 nRanks 96 busId 43000
e07r2n18:25919:26914 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n18:25919:26914 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n01:627:1821 [1] NCCL INFO Channel 00 : 13[26000] -> 12[4000] via P2P/IPC comm 0x2b75e4000ab0 nRanks 96
e07r2n09:23025:24185 [2] NCCL INFO Connected all trees comm 0x2b44d8000ab0 nRanks 96 busId 43000
e07r2n01:627:1821 [1] NCCL INFO Channel 01 : 13[26000] -> 12[4000] via P2P/IPC comm 0x2b75e4000ab0 nRanks 96
e07r2n09:23025:24185 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n09:23025:24185 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n10:29372:30328 [2] NCCL INFO Connected all trees comm 0x2b4700000ab0 nRanks 96 busId 43000
e07r2n10:29372:30328 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n10:29372:30328 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n17:31185:32201 [1] NCCL INFO Channel 00 : 77[26000] -> 76[4000] via P2P/IPC comm 0x2ac6ec000ab0 nRanks 96
e07r2n17:31185:32201 [1] NCCL INFO Channel 01 : 77[26000] -> 76[4000] via P2P/IPC comm 0x2ac6ec000ab0 nRanks 96
e07r2n01:628:1825 [2] NCCL INFO Connected all trees comm 0x2ae908000ab0 nRanks 96 busId 43000
e07r2n01:628:1825 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n01:628:1825 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n02:4758:5857 [1] NCCL INFO Channel 00 : 17[26000] -> 16[4000] via P2P/IPC comm 0x2b8424000ab0 nRanks 96
e07r2n02:4758:5857 [1] NCCL INFO Channel 01 : 17[26000] -> 16[4000] via P2P/IPC comm 0x2b8424000ab0 nRanks 96
e07r2n17:31186:32202 [2] NCCL INFO Connected all trees comm 0x2b2e7c000ab0 nRanks 96 busId 43000
e07r2n17:31186:32202 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31186:32202 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n02:4759:5859 [2] NCCL INFO Connected all trees comm 0x2b32b8000ab0 nRanks 96 busId 43000
e07r2n02:4759:5859 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4759:5859 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n02:4756:5860 [0] NCCL INFO Channel 00/0 : 33[26000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b6f54000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 00/0 : 57[26000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b8468000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 00/0 : 64[4000] -> 0[4000] [send] via NET/IB/0 comm 0x2b696c000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 00/0 : 32[4000] -> 65[26000] [send] via NET/IB/0 comm 0x2b2990000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 00/0 : 25[26000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3b20000ab0 nRanks 96
e07r2n06:26609:27563 [1] NCCL INFO Channel 00/0 : 33[26000] -> 16[4000] [send] via NET/IB/0 comm 0x2ae710000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 00/0 : 80[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b9534000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 00/0 : 48[4000] -> 32[4000] [send] via NET/IB/0 comm 0x2b24f0000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 00/0 : 89[26000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5860000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 01/0 : 60[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af760000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 00/0 : 44[4000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4008000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 00/0 : 76[4000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7c84000ab0 nRanks 96
e07r2n13:31207:32206 [1] NCCL INFO Channel 01/0 : 76[4000] -> 61[26000] [receive] via NET/IB/0 comm 0x2aeb40000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 01/0 : 76[4000] -> 61[26000] [send] via NET/IB/0 comm 0x2baf30000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 01/0 : 29[26000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ae9c8000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 01/0 : 28[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae748000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 00/0 : 12[4000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7be0000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 01/0 : 92[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba3ac000ab0 nRanks 96
e07r2n12:16037:17013 [1] NCCL INFO Channel 00 : 57[26000] -> 56[4000] via P2P/IPC comm 0x2ae0f4000ab0 nRanks 96
e07r2n12:16037:17013 [1] NCCL INFO Channel 01 : 57[26000] -> 56[4000] via P2P/IPC comm 0x2ae0f4000ab0 nRanks 96
e07r2n06:26609:27563 [1] NCCL INFO Channel 00 : 33[26000] -> 32[4000] via P2P/IPC comm 0x2ae710000ab0 nRanks 96
e07r2n06:26609:27563 [1] NCCL INFO Channel 01 : 33[26000] -> 32[4000] via P2P/IPC comm 0x2ae710000ab0 nRanks 96
e07r2n04:1827:3073 [1] NCCL INFO Channel 00 : 25[26000] -> 24[4000] via P2P/IPC comm 0x2abfb4000ab0 nRanks 96
e07r2n04:1827:3073 [1] NCCL INFO Channel 01 : 25[26000] -> 24[4000] via P2P/IPC comm 0x2abfb4000ab0 nRanks 96
e07r2n12:16038:17016 [2] NCCL INFO Connected all trees comm 0x2b93f8000ab0 nRanks 96 busId 43000
e07r2n12:16038:17016 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n12:16038:17016 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n06:26610:27569 [2] NCCL INFO Connected all trees comm 0x2b9324000ab0 nRanks 96 busId 43000
e07r2n06:26610:27569 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n06:26610:27569 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n04:1828:3077 [2] NCCL INFO Connected all trees comm 0x2b200c000ab0 nRanks 96 busId 43000
e07r2n04:1828:3077 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n04:1828:3077 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24491:25708 [1] NCCL INFO Channel 00 : 89[26000] -> 88[4000] via P2P/IPC comm 0x2b9b5c000ab0 nRanks 96
e07r3n00:24491:25708 [1] NCCL INFO Channel 01 : 89[26000] -> 88[4000] via P2P/IPC comm 0x2b9b5c000ab0 nRanks 96
e07r3n00:24492:25711 [2] NCCL INFO Connected all trees comm 0x2b96c4000ab0 nRanks 96 busId 43000
e07r3n00:24492:25711 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n00:24492:25711 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n13:31207:32206 [1] NCCL INFO Channel 00 : 61[26000] -> 60[4000] via P2P/IPC comm 0x2aeb40000ab0 nRanks 96
e07r2n13:31207:32206 [1] NCCL INFO Channel 01 : 61[26000] -> 60[4000] via P2P/IPC comm 0x2aeb40000ab0 nRanks 96
e07r2n13:31209:32208 [2] NCCL INFO Connected all trees comm 0x2ab7c4000ab0 nRanks 96 busId 43000
e07r2n13:31209:32208 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31209:32208 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22275:23497 [0] NCCL INFO Channel 00/0 : 0[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b696c000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 00/0 : 65[26000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2990000ab0 nRanks 96
e07r2n11:16526:17682 [0] NCCL INFO Channel 01/0 : 52[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2b8468000ab0 nRanks 96
e07r2n14:22276:23489 [1] NCCL INFO Channel 00/0 : 65[26000] -> 32[4000] [send] via NET/IB/0 comm 0x2aee9c000ab0 nRanks 96
e07r1n18:15185:16349 [0] NCCL INFO Channel 00/0 : 0[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b2c88000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Channel 01/0 : 20[4000] -> 16[4000] [send] via NET/IB/0 comm 0x2b3b20000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 00/0 : 24[4000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b6f54000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Channel 01/0 : 84[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b5860000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 01/0 : 76[4000] -> 68[4000] [send] via NET/IB/0 comm 0x2baf30000ab0 nRanks 96
e07r2n05:907:2100 [1] NCCL INFO Channel 01/0 : 60[4000] -> 29[26000] [receive] via NET/IB/0 comm 0x2af6c0000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 01/0 : 60[4000] -> 29[26000] [send] via NET/IB/0 comm 0x2ae9c8000ab0 nRanks 96
e07r3n01:6046:7322 [0] NCCL INFO Channel 01/0 : 28[4000] -> 92[4000] [receive] via NET/IB/0 comm 0x2b0b68000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 01/0 : 28[4000] -> 92[4000] [send] via NET/IB/0 comm 0x2ba3ac000ab0 nRanks 96
e07r2n14:22276:23489 [1] NCCL INFO Channel 00 : 65[26000] -> 64[4000] via P2P/IPC comm 0x2aee9c000ab0 nRanks 96
e07r2n14:22276:23489 [1] NCCL INFO Channel 01 : 65[26000] -> 64[4000] via P2P/IPC comm 0x2aee9c000ab0 nRanks 96
e07r2n14:22277:23498 [2] NCCL INFO Connected all trees comm 0x2b736c000ab0 nRanks 96 busId 43000
e07r2n14:22277:23498 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n14:22277:23498 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n05:907:2100 [1] NCCL INFO Channel 00 : 29[26000] -> 28[4000] via P2P/IPC comm 0x2af6c0000ab0 nRanks 96
e07r2n05:907:2100 [1] NCCL INFO Channel 01 : 29[26000] -> 28[4000] via P2P/IPC comm 0x2af6c0000ab0 nRanks 96
e07r2n05:908:2101 [2] NCCL INFO Connected all trees comm 0x2b0160000ab0 nRanks 96 busId 43000
e07r2n05:908:2101 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:908:2101 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n18:15185:16349 [0] NCCL INFO Channel 01/0 : 4[4000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b2c88000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 00/0 : 28[4000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b09f8000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 00/0 : 80[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b696c000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 00/0 : 48[4000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2990000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Channel 00/0 : 76[4000] -> 72[4000] [send] via NET/IB/0 comm 0x2baf30000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Channel 01/0 : 20[4000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b6f54000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 01/0 : 60[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2ae9c8000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 00/0 : 73[26000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b1e30000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 01/0 : 28[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2ba3ac000ab0 nRanks 96
e07r3n01:6046:7322 [0] NCCL INFO Channel 00/0 : 92[4000] -> 88[4000] [send] via NET/IB/0 comm 0x2b0b68000ab0 nRanks 96
e07r2n17:31184:32198 [0] NCCL INFO Connected all trees comm 0x2baf30000ab0 nRanks 96 busId 4000
e07r2n17:31184:32198 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31184:32198 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n17:31185:32201 [1] NCCL INFO Connected all trees comm 0x2ac6ec000ab0 nRanks 96 busId 26000
e07r2n17:31185:32201 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31185:32201 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26996:28158 [1] NCCL INFO Channel 00 : 73[26000] -> 72[4000] via P2P/IPC comm 0x2ade8c000ab0 nRanks 96
e07r2n16:26996:28158 [1] NCCL INFO Channel 01 : 73[26000] -> 72[4000] via P2P/IPC comm 0x2ade8c000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO Connected all trees comm 0x2b6f54000ab0 nRanks 96 busId 4000
e07r2n02:4756:5860 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4756:5860 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n02:4758:5857 [1] NCCL INFO Connected all trees comm 0x2b8424000ab0 nRanks 96 busId 26000
e07r2n02:4758:5857 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4758:5857 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26997:28160 [2] NCCL INFO Connected all trees comm 0x2b2168000ab0 nRanks 96 busId 43000
e07r2n16:26997:28160 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n16:26997:28160 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n18:25917:26912 [0] NCCL INFO Channel 00/0 : 88[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b9534000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 00/0 : 56[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b24f0000ab0 nRanks 96
e07r2n14:22275:23497 [0] NCCL INFO Channel 01/0 : 68[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b696c000ab0 nRanks 96
e07r2n13:31206:32207 [0] NCCL INFO Channel 00/0 : 60[4000] -> 56[4000] [send] via NET/IB/0 comm 0x2ae9c8000ab0 nRanks 96
e07r2n06:26608:27568 [0] NCCL INFO Channel 01/0 : 36[4000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2990000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Channel 01/0 : 72[4000] -> 69[26000] [send] via NET/IB/0 comm 0x2b7c84000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 01/0 : 44[4000] -> 36[4000] [send] via NET/IB/0 comm 0x2af760000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Channel 01/0 : 68[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b1e30000ab0 nRanks 96
e07r2n05:906:2099 [0] NCCL INFO Channel 00/0 : 28[4000] -> 24[4000] [send] via NET/IB/0 comm 0x2ba3ac000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 01/0 : 12[4000] -> 4[4000] [send] via NET/IB/0 comm 0x2ae748000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO Connected all trees comm 0x2b7c84000ab0 nRanks 96 busId 4000
e07r2n16:26995:28159 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n16:26995:28159 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26996:28158 [1] NCCL INFO Connected all trees comm 0x2ade8c000ab0 nRanks 96 busId 26000
e07r2n16:26996:28158 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n16:26996:28158 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28147:29326 [1] NCCL INFO Channel 00 : 69[26000] -> 68[4000] via P2P/IPC comm 0x2b19d0000ab0 nRanks 96
e07r2n15:28147:29326 [1] NCCL INFO Channel 01 : 69[26000] -> 68[4000] via P2P/IPC comm 0x2b19d0000ab0 nRanks 96
e07r2n15:28145:29327 [0] NCCL INFO Connected all trees comm 0x2b1e30000ab0 nRanks 96 busId 4000
e07r2n15:28145:29327 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28145:29327 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28148:29328 [2] NCCL INFO Connected all trees comm 0x2afa50000ab0 nRanks 96 busId 43000
e07r2n15:28147:29326 [1] NCCL INFO Connected all trees comm 0x2b19d0000ab0 nRanks 96 busId 26000
e07r2n15:28148:29328 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28148:29328 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28147:29326 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28147:29326 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n05:906:2099 [0] NCCL INFO Connected all trees comm 0x2ba3ac000ab0 nRanks 96 busId 4000
e07r2n05:906:2099 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:906:2099 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n05:907:2100 [1] NCCL INFO Connected all trees comm 0x2af6c0000ab0 nRanks 96 busId 26000
e07r2n05:907:2100 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:907:2100 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22275:23497 [0] NCCL INFO Connected all trees comm 0x2b696c000ab0 nRanks 96 busId 4000
e07r2n14:22275:23497 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n14:22275:23497 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22276:23489 [1] NCCL INFO Connected all trees comm 0x2aee9c000ab0 nRanks 96 busId 26000
e07r2n14:22276:23489 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n14:22276:23489 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24490:25707 [0] NCCL INFO Channel 00/0 : 92[4000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b84fc000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 00/0 : 60[4000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b831c000ab0 nRanks 96
e07r2n18:25917:26912 [0] NCCL INFO Channel 01/0 : 84[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b9534000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Channel 01/0 : 52[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b24f0000ab0 nRanks 96
e07r2n09:23023:24187 [0] NCCL INFO Channel 00/0 : 44[4000] -> 40[4000] [send] via NET/IB/0 comm 0x2af760000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 00/0 : 41[26000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac774000ab0 nRanks 96
e07r2n04:1826:3076 [0] NCCL INFO Channel 01/0 : 24[4000] -> 21[26000] [send] via NET/IB/0 comm 0x2b09f8000ab0 nRanks 96
e07r2n01:626:1823 [0] NCCL INFO Channel 00/0 : 12[4000] -> 8[4000] [send] via NET/IB/0 comm 0x2ae748000ab0 nRanks 96
e07r2n02:4756:5860 [0] NCCL INFO comm 0x2b6f54000ab0 rank 16 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n02:4758:5857 [1] NCCL INFO comm 0x2b8424000ab0 rank 17 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n02:4760:5858 [3] NCCL INFO comm 0x2b72e8000ab0 rank 19 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n19:30599:31677 [0] NCCL INFO Channel 00/0 : 9[26000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b6e1c000ab0 nRanks 96
e07r2n17:31186:32202 [2] NCCL INFO comm 0x2b2e7c000ab0 rank 78 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n17:31184:32198 [0] NCCL INFO comm 0x2baf30000ab0 rank 76 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n17:31187:32200 [3] NCCL INFO comm 0x2b1110000ab0 rank 79 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n17:31185:32201 [1] NCCL INFO comm 0x2ac6ec000ab0 rank 77 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n02:4759:5859 [2] NCCL INFO comm 0x2b32b8000ab0 rank 18 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n01:6046:7322 [0] NCCL INFO Connected all trees comm 0x2b0b68000ab0 nRanks 96 busId 4000
e07r3n01:6046:7322 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6046:7322 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n01:6047:7323 [1] NCCL INFO Connected all trees comm 0x2adc2c000ab0 nRanks 96 busId 26000
e07r3n01:6047:7323 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6047:7323 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n13:31206:32207 [0] NCCL INFO Connected all trees comm 0x2ae9c8000ab0 nRanks 96 busId 4000
e07r2n13:31206:32207 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31206:32207 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n13:31207:32206 [1] NCCL INFO Connected all trees comm 0x2aeb40000ab0 nRanks 96 busId 26000
e07r2n13:31207:32206 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31207:32206 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n09:23023:24187 [0] NCCL INFO Connected all trees comm 0x2af760000ab0 nRanks 96 busId 4000
e07r2n09:23023:24187 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n09:23023:24187 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n09:23024:24184 [1] NCCL INFO Connected all trees comm 0x2b39f0000ab0 nRanks 96 busId 26000
e07r2n09:23024:24184 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n09:23024:24184 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n18:25917:26912 [0] NCCL INFO Connected all trees comm 0x2b9534000ab0 nRanks 96 busId 4000
e07r2n18:25917:26912 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n18:25917:26912 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n18:25918:26911 [1] NCCL INFO Connected all trees comm 0x2b094c000ab0 nRanks 96 busId 26000
e07r2n18:25918:26911 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n18:25918:26911 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n08:30378:31693 [1] NCCL INFO Channel 00 : 41[26000] -> 40[4000] via P2P/IPC comm 0x2b2bac000ab0 nRanks 96
e07r2n08:30378:31693 [1] NCCL INFO Channel 01 : 41[26000] -> 40[4000] via P2P/IPC comm 0x2b2bac000ab0 nRanks 96
e07r2n10:29369:30329 [0] NCCL INFO Connected all trees comm 0x2b24f0000ab0 nRanks 96 busId 4000
e07r2n10:29369:30329 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n10:29369:30329 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n10:29371:30326 [1] NCCL INFO Connected all trees comm 0x2ad4c4000ab0 nRanks 96 busId 26000
e07r2n10:29371:30326 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n10:29371:30326 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n08:30379:31736 [2] NCCL INFO Connected all trees comm 0x2add50000ab0 nRanks 96 busId 43000
e07r2n08:30379:31736 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n08:30379:31736 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n01:626:1823 [0] NCCL INFO Connected all trees comm 0x2ae748000ab0 nRanks 96 busId 4000
e07r2n01:626:1823 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n01:626:1823 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n04:1826:3076 [0] NCCL INFO Connected all trees comm 0x2b09f8000ab0 nRanks 96 busId 4000
e07r2n04:1826:3076 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n04:1826:3076 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n04:1827:3073 [1] NCCL INFO Connected all trees comm 0x2abfb4000ab0 nRanks 96 busId 26000
e07r2n04:1827:3073 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n04:1827:3073 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n01:627:1821 [1] NCCL INFO Connected all trees comm 0x2b75e4000ab0 nRanks 96 busId 26000
e07r2n01:627:1821 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n01:627:1821 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n03:21864:23049 [1] NCCL INFO Channel 00 : 21[26000] -> 20[4000] via P2P/IPC comm 0x2b3270000ab0 nRanks 96
e07r2n03:21864:23049 [1] NCCL INFO Channel 01 : 21[26000] -> 20[4000] via P2P/IPC comm 0x2b3270000ab0 nRanks 96
e07r2n03:21863:23045 [0] NCCL INFO Connected all trees comm 0x2b3b20000ab0 nRanks 96 busId 4000
e07r2n03:21863:23045 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n03:21863:23045 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n03:21865:23050 [2] NCCL INFO Connected all trees comm 0x2b601c000ab0 nRanks 96 busId 43000
e07r2n03:21864:23049 [1] NCCL INFO Connected all trees comm 0x2b3270000ab0 nRanks 96 busId 26000
e07r2n03:21865:23050 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n03:21865:23050 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n03:21864:23049 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n03:21864:23049 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23101:24353 [1] NCCL INFO Channel 00 : 9[26000] -> 8[4000] via P2P/IPC comm 0x2abe94000ab0 nRanks 96
e07r2n00:23101:24353 [1] NCCL INFO Channel 01 : 9[26000] -> 8[4000] via P2P/IPC comm 0x2abe94000ab0 nRanks 96
e07r2n00:23102:24358 [2] NCCL INFO Connected all trees comm 0x2acbf4000ab0 nRanks 96 busId 43000
e07r2n00:23102:24358 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23102:24358 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24490:25707 [0] NCCL INFO Channel 01/0 : 88[4000] -> 85[26000] [send] via NET/IB/0 comm 0x2b84fc000ab0 nRanks 96
e07r2n12:16035:17011 [0] NCCL INFO Channel 01/0 : 56[4000] -> 53[26000] [send] via NET/IB/0 comm 0x2b831c000ab0 nRanks 96
e07r2n08:30377:31697 [0] NCCL INFO Channel 01/0 : 40[4000] -> 37[26000] [send] via NET/IB/0 comm 0x2b4008000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Channel 01/0 : 36[4000] -> 32[4000] [send] via NET/IB/0 comm 0x2ac774000ab0 nRanks 96
e07r2n00:23099:24278 [0] NCCL INFO Channel 01/0 : 8[4000] -> 5[26000] [send] via NET/IB/0 comm 0x2b7be0000ab0 nRanks 96
e07r2n15:28147:29326 [1] NCCL INFO comm 0x2b19d0000ab0 rank 69 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n15:28150:29329 [3] NCCL INFO comm 0x2b3a44000ab0 rank 71 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n05:906:2099 [0] NCCL INFO comm 0x2ba3ac000ab0 rank 28 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n05:908:2101 [2] NCCL INFO comm 0x2b0160000ab0 rank 30 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r1n19:30599:31677 [0] NCCL INFO Channel 01/0 : 4[4000] -> 0[4000] [send] via NET/IB/0 comm 0x2b6e1c000ab0 nRanks 96
e07r2n16:26995:28159 [0] NCCL INFO comm 0x2b7c84000ab0 rank 72 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n16:26997:28160 [2] NCCL INFO comm 0x2b2168000ab0 rank 74 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n16:26998:28161 [3] NCCL INFO comm 0x2b5c98000ab0 rank 75 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n16:26996:28158 [1] NCCL INFO comm 0x2ade8c000ab0 rank 73 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n00:24490:25707 [0] NCCL INFO Connected all trees comm 0x2b84fc000ab0 nRanks 96 busId 4000
e07r3n00:24490:25707 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n00:24490:25707 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24491:25708 [1] NCCL INFO Connected all trees comm 0x2b9b5c000ab0 nRanks 96 busId 26000
e07r3n00:24491:25708 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n00:24491:25708 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n12:16035:17011 [0] NCCL INFO Connected all trees comm 0x2b831c000ab0 nRanks 96 busId 4000
e07r2n12:16035:17011 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28148:29328 [2] NCCL INFO comm 0x2afa50000ab0 rank 70 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n12:16035:17011 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28145:29327 [0] NCCL INFO comm 0x2b1e30000ab0 rank 68 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n12:16037:17013 [1] NCCL INFO Connected all trees comm 0x2ae0f4000ab0 nRanks 96 busId 26000
e07r2n12:16037:17013 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n12:16037:17013 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n05:907:2100 [1] NCCL INFO comm 0x2af6c0000ab0 rank 29 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n05:909:2102 [3] NCCL INFO comm 0x2ac2e4000ab0 rank 31 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n14:22277:23498 [2] NCCL INFO comm 0x2b736c000ab0 rank 66 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n14:22275:23497 [0] NCCL INFO comm 0x2b696c000ab0 rank 64 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n19:2109:3517 [1] NCCL INFO Channel 00 : 85[26000] -> 84[4000] via P2P/IPC comm 0x2b2e10000ab0 nRanks 96
e07r2n19:2109:3517 [1] NCCL INFO Channel 01 : 85[26000] -> 84[4000] via P2P/IPC comm 0x2b2e10000ab0 nRanks 96
e07r2n11:16527:17685 [1] NCCL INFO Channel 00 : 53[26000] -> 52[4000] via P2P/IPC comm 0x2ae98c000ab0 nRanks 96
e07r2n14:22276:23489 [1] NCCL INFO comm 0x2aee9c000ab0 rank 65 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n14:22278:23496 [3] NCCL INFO comm 0x2b1038000ab0 rank 67 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n11:16527:17685 [1] NCCL INFO Channel 01 : 53[26000] -> 52[4000] via P2P/IPC comm 0x2ae98c000ab0 nRanks 96
e07r2n19:2107:3519 [0] NCCL INFO Connected all trees comm 0x2b5860000ab0 nRanks 96 busId 4000
e07r2n19:2107:3519 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n19:2107:3519 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n19:2109:3517 [1] NCCL INFO Connected all trees comm 0x2b2e10000ab0 nRanks 96 busId 26000
e07r2n19:2110:3521 [2] NCCL INFO Connected all trees comm 0x2b9ce8000ab0 nRanks 96 busId 43000
e07r2n19:2109:3517 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n19:2109:3517 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n19:2110:3521 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n19:2110:3521 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16526:17682 [0] NCCL INFO Connected all trees comm 0x2b8468000ab0 nRanks 96 busId 4000
e07r2n11:16526:17682 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16526:17682 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16528:17686 [2] NCCL INFO Connected all trees comm 0x2b8a80000ab0 nRanks 96 busId 43000
e07r2n11:16528:17686 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16527:17685 [1] NCCL INFO Connected all trees comm 0x2ae98c000ab0 nRanks 96 busId 26000
e07r2n11:16528:17686 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16527:17685 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16527:17685 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n08:30377:31697 [0] NCCL INFO Connected all trees comm 0x2b4008000ab0 nRanks 96 busId 4000
e07r2n08:30377:31697 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n08:30377:31697 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n08:30378:31693 [1] NCCL INFO Connected all trees comm 0x2b2bac000ab0 nRanks 96 busId 26000
e07r2n08:30378:31693 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n08:30378:31693 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n07:11608:12938 [1] NCCL INFO Channel 00 : 37[26000] -> 36[4000] via P2P/IPC comm 0x2aeaa8000ab0 nRanks 96
e07r2n07:11608:12938 [1] NCCL INFO Channel 01 : 37[26000] -> 36[4000] via P2P/IPC comm 0x2aeaa8000ab0 nRanks 96
e07r2n07:11607:12941 [0] NCCL INFO Connected all trees comm 0x2ac774000ab0 nRanks 96 busId 4000
e07r2n07:11607:12941 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11607:12941 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n07:11609:12944 [2] NCCL INFO Connected all trees comm 0x2aca80000ab0 nRanks 96 busId 43000
e07r2n07:11608:12938 [1] NCCL INFO Connected all trees comm 0x2aeaa8000ab0 nRanks 96 busId 26000
e07r2n07:11609:12944 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11609:12944 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n07:11608:12938 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11608:12938 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23099:24278 [0] NCCL INFO Connected all trees comm 0x2b7be0000ab0 nRanks 96 busId 4000
e07r2n00:23099:24278 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23099:24278 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n06:26608:27568 [0] NCCL INFO Connected all trees comm 0x2b2990000ab0 nRanks 96 busId 4000
e07r2n06:26608:27568 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23101:24353 [1] NCCL INFO Connected all trees comm 0x2abe94000ab0 nRanks 96 busId 26000
e07r2n06:26608:27568 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23101:24353 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23101:24353 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n06:26609:27563 [1] NCCL INFO Connected all trees comm 0x2ae710000ab0 nRanks 96 busId 26000
e07r2n06:26609:27563 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n06:26609:27563 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n19:30600:31684 [1] NCCL INFO Channel 00 : 5[26000] -> 4[4000] via P2P/IPC comm 0x2b4630000ab0 nRanks 96
e07r1n19:30600:31684 [1] NCCL INFO Channel 01 : 5[26000] -> 4[4000] via P2P/IPC comm 0x2b4630000ab0 nRanks 96
e07r1n19:30599:31677 [0] NCCL INFO Connected all trees comm 0x2b6e1c000ab0 nRanks 96 busId 4000
e07r1n19:30599:31677 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n19:30599:31677 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n19:30601:31683 [2] NCCL INFO Connected all trees comm 0x2b9508000ab0 nRanks 96 busId 43000
e07r1n19:30600:31684 [1] NCCL INFO Connected all trees comm 0x2b4630000ab0 nRanks 96 busId 26000
e07r1n19:30601:31683 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n19:30601:31683 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n19:30600:31684 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n19:30600:31684 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n18:15185:16349 [0] NCCL INFO Connected all trees comm 0x2b2c88000ab0 nRanks 96 busId 4000
e07r1n18:15185:16349 [0] NCCL INFO Using tuning table 0 with LL128 disabled
e07r1n18:15185:16349 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n18:15185:16349 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n18:15187:16550 [1] NCCL INFO Connected all trees comm 0x2ac740000ab0 nRanks 96 busId 26000
e07r1n18:15187:16550 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n18:15187:16550 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n01:6048:7326 [2] NCCL INFO comm 0x2b3f2c000ab0 rank 94 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n01:6047:7323 [1] NCCL INFO comm 0x2adc2c000ab0 rank 93 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n13:31206:32207 [0] NCCL INFO comm 0x2ae9c8000ab0 rank 60 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n13:31209:32208 [2] NCCL INFO comm 0x2ab7c4000ab0 rank 62 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n09:23025:24185 [2] NCCL INFO comm 0x2b44d8000ab0 rank 46 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n09:23024:24184 [1] NCCL INFO comm 0x2b39f0000ab0 rank 45 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n18:25917:26912 [0] NCCL INFO comm 0x2b9534000ab0 rank 80 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n18:25920:26909 [3] NCCL INFO comm 0x2af5b8000ab0 rank 83 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n18:25918:26911 [1] NCCL INFO comm 0x2b094c000ab0 rank 81 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n01:6046:7322 [0] NCCL INFO comm 0x2b0b68000ab0 rank 92 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n10:29372:30328 [2] NCCL INFO comm 0x2b4700000ab0 rank 50 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n10:29369:30329 [0] NCCL INFO comm 0x2b24f0000ab0 rank 48 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n10:29371:30326 [1] NCCL INFO comm 0x2ad4c4000ab0 rank 49 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n01:6049:7320 [3] NCCL INFO comm 0x2b4e20000ab0 rank 95 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n04:1828:3077 [2] NCCL INFO comm 0x2b200c000ab0 rank 26 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n13:31207:32206 [1] NCCL INFO comm 0x2aeb40000ab0 rank 61 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n04:1829:3075 [3] NCCL INFO comm 0x2b663c000ab0 rank 27 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n01:627:1821 [1] NCCL INFO comm 0x2b75e4000ab0 rank 13 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n13:31210:32209 [3] NCCL INFO comm 0x2af840000ab0 rank 63 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n01:626:1823 [0] NCCL INFO comm 0x2ae748000ab0 rank 12 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n03:21866:23047 [3] NCCL INFO comm 0x2b6bd4000ab0 rank 23 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n01:628:1825 [2] NCCL INFO comm 0x2ae908000ab0 rank 14 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n09:23023:24187 [0] NCCL INFO comm 0x2af760000ab0 rank 44 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n09:23026:24186 [3] NCCL INFO comm 0x2b48e0000ab0 rank 47 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n18:25919:26914 [2] NCCL INFO comm 0x2aef38000ab0 rank 82 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n03:21864:23049 [1] NCCL INFO comm 0x2b3270000ab0 rank 21 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n10:29373:30327 [3] NCCL INFO comm 0x2b2b38000ab0 rank 51 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n04:1826:3076 [0] NCCL INFO comm 0x2b09f8000ab0 rank 24 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n01:629:1819 [3] NCCL INFO comm 0x2ace88000ab0 rank 15 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n04:1827:3073 [1] NCCL INFO comm 0x2abfb4000ab0 rank 25 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n03:21863:23045 [0] NCCL INFO comm 0x2b3b20000ab0 rank 20 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n03:21865:23050 [2] NCCL INFO comm 0x2b601c000ab0 rank 22 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n12:16035:17011 [0] NCCL INFO comm 0x2b831c000ab0 rank 56 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n12:16037:17013 [1] NCCL INFO comm 0x2ae0f4000ab0 rank 57 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n11:16528:17686 [2] NCCL INFO comm 0x2b8a80000ab0 rank 54 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n19:2107:3519 [0] NCCL INFO comm 0x2b5860000ab0 rank 84 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n19:2110:3521 [2] NCCL INFO comm 0x2b9ce8000ab0 rank 86 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n08:30377:31697 [0] NCCL INFO comm 0x2b4008000ab0 rank 40 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n08:30380:31690 [3] NCCL INFO comm 0x2b85a8000ab0 rank 43 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n12:16039:17014 [3] NCCL INFO comm 0x2b3ec8000ab0 rank 59 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r3n00:24492:25711 [2] NCCL INFO comm 0x2b96c4000ab0 rank 90 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n07:11608:12938 [1] NCCL INFO comm 0x2aeaa8000ab0 rank 37 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n00:24493:25710 [3] NCCL INFO comm 0x2b061c000ab0 rank 91 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r3n00:24491:25708 [1] NCCL INFO comm 0x2b9b5c000ab0 rank 89 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n12:16038:17016 [2] NCCL INFO comm 0x2b93f8000ab0 rank 58 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n00:24490:25707 [0] NCCL INFO comm 0x2b84fc000ab0 rank 88 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n06:26608:27568 [0] NCCL INFO comm 0x2b2990000ab0 rank 32 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n06:26610:27569 [2] NCCL INFO comm 0x2b9324000ab0 rank 34 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n00:23099:24278 [0] NCCL INFO comm 0x2b7be0000ab0 rank 8 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n06:26609:27563 [1] NCCL INFO comm 0x2ae710000ab0 rank 33 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n00:23103:24289 [3] NCCL INFO comm 0x2b8398000ab0 rank 11 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n19:2111:3520 [3] NCCL INFO comm 0x2b30c8000ab0 rank 87 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n08:30379:31736 [2] NCCL INFO comm 0x2add50000ab0 rank 42 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n19:2109:3517 [1] NCCL INFO comm 0x2b2e10000ab0 rank 85 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n08:30378:31693 [1] NCCL INFO comm 0x2b2bac000ab0 rank 41 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n11:16529:17687 [3] NCCL INFO comm 0x2b82f0000ab0 rank 55 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n11:16527:17685 [1] NCCL INFO comm 0x2ae98c000ab0 rank 53 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n11:16526:17682 [0] NCCL INFO comm 0x2b8468000ab0 rank 52 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n18:15188:16551 [2] NCCL INFO comm 0x2b6a08000ab0 rank 2 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r1n18:15187:16550 [1] NCCL INFO comm 0x2ac740000ab0 rank 1 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n07:11610:12943 [3] NCCL INFO comm 0x2afd74000ab0 rank 39 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n00:23102:24358 [2] NCCL INFO comm 0x2acbf4000ab0 rank 10 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n07:11607:12941 [0] NCCL INFO comm 0x2ac774000ab0 rank 36 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n07:11609:12944 [2] NCCL INFO comm 0x2aca80000ab0 rank 38 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n00:23101:24353 [1] NCCL INFO comm 0x2abe94000ab0 rank 9 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n06:26611:27566 [3] NCCL INFO comm 0x2b6078000ab0 rank 35 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n19:30602:31679 [3] NCCL INFO comm 0x2b2b9c000ab0 rank 7 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n19:30600:31684 [1] NCCL INFO comm 0x2b4630000ab0 rank 5 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r1n19:30601:31683 [2] NCCL INFO comm 0x2b9508000ab0 rank 6 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r1n19:30599:31677 [0] NCCL INFO comm 0x2b6e1c000ab0 rank 4 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n18:15189:16547 [3] NCCL INFO comm 0x2b6d50000ab0 rank 3 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n18:15185:16349 [0] NCCL INFO comm 0x2b2c88000ab0 rank 0 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
[2023-11-09 12:05:55,620] [INFO] [partition_parameters.py:454:__exit__] finished initializing model with 7.51B parameters
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
Parameter Offload: Total persistent parameters: 266240 in 65 params
e07r1n18:15185:19054 [0] NCCL INFO Using network IB
e07r1n18:15189:19057 [3] NCCL INFO Using network IB
e07r1n18:15188:19056 [2] NCCL INFO Using network IB
e07r3n01:6048:9746 [2] NCCL INFO Using network IB
e07r2n19:2110:5599 [2] NCCL INFO Using network IB
e07r2n18:25919:28639 [2] NCCL INFO Using network IB
e07r2n04:1828:5186 [2] NCCL INFO Using network IB
e07r2n10:29372:32186 [2] NCCL INFO Using network IB
e07r2n01:628:4325 [2] NCCL INFO Using network IB
e07r2n08:30379:1323 [2] NCCL INFO Using network IB
e07r2n16:26998:30253 [3] NCCL INFO Using network IB
e07r2n18:25920:28641 [3] NCCL INFO Using network IB
e07r2n08:30380:1325 [3] NCCL INFO Using network IB
e07r2n16:26997:30254 [2] NCCL INFO Using network IB
e07r2n12:16038:18764 [2] NCCL INFO Using network IB
e07r2n02:4759:8314 [2] NCCL INFO Using network IB
e07r2n07:11609:15048 [2] NCCL INFO Using network IB
e07r2n05:909:4280 [3] NCCL INFO Using network IB
e07r2n03:21865:25084 [2] NCCL INFO Using network IB
e07r2n12:16039:18766 [3] NCCL INFO Using network IB
e07r2n02:4760:8316 [3] NCCL INFO Using network IB
e07r2n07:11610:15050 [3] NCCL INFO Using network IB
e07r2n05:908:4281 [2] NCCL INFO Using network IB
e07r2n03:21866:25086 [3] NCCL INFO Using network IB
e07r1n19:30602:1034 [3] NCCL INFO Using network IB
e07r2n06:26610:29330 [2] NCCL INFO Using network IB
e07r2n01:629:4327 [3] NCCL INFO Using network IB
e07r2n00:23102:26350 [2] NCCL INFO Using network IB
e07r2n17:31186:1568 [2] NCCL INFO Using network IB
e07r2n15:28148:31326 [2] NCCL INFO Using network IB
e07r3n00:24492:27773 [2] NCCL INFO Using network IB
e07r3n01:6049:9748 [3] NCCL INFO Using network IB
e07r2n04:1829:5188 [3] NCCL INFO Using network IB
e07r2n10:29373:32188 [3] NCCL INFO Using network IB
e07r2n19:2111:5601 [3] NCCL INFO Using network IB
e07r2n14:22278:25514 [3] NCCL INFO Using network IB
e07r1n18:15187:19060 [1] NCCL INFO Using network IB
e07r2n12:16035:18767 [0] NCCL INFO Using network IB
e07r2n02:4756:8318 [0] NCCL INFO Using network IB
e07r2n05:906:4284 [0] NCCL INFO Using network IB
e07r1n19:30601:1035 [2] NCCL INFO Using network IB
e07r2n06:26611:29331 [3] NCCL INFO Using network IB
e07r2n01:626:4329 [0] NCCL INFO Using network IB
e07r2n00:23103:26351 [3] NCCL INFO Using network IB
e07r2n17:31187:1570 [3] NCCL INFO Using network IB
e07r2n15:28150:31327 [3] NCCL INFO Using network IB
e07r3n00:24493:27774 [3] NCCL INFO Using network IB
e07r3n01:6046:9750 [0] NCCL INFO Using network IB
e07r2n04:1826:5190 [0] NCCL INFO Using network IB
e07r2n10:29369:32190 [0] NCCL INFO Using network IB
e07r2n19:2107:5602 [0] NCCL INFO Using network IB
e07r2n14:22277:25513 [2] NCCL INFO Using network IB
e07r2n16:26995:30257 [0] NCCL INFO Using network IB
e07r2n11:16529:19772 [3] NCCL INFO Using network IB
e07r2n09:23026:26303 [3] NCCL INFO Using network IB
e07r2n08:30377:1327 [0] NCCL INFO Using network IB
e07r2n18:25917:28643 [0] NCCL INFO Using network IB
e07r2n12:16037:18770 [1] NCCL INFO Using network IB
e07r2n02:4758:8319 [1] NCCL INFO Using network IB
e07r2n05:907:4286 [1] NCCL INFO Using network IB
e07r1n19:30599:1037 [0] NCCL INFO Using network IB
e07r2n06:26608:29334 [0] NCCL INFO Using network IB
e07r2n01:627:4330 [1] NCCL INFO Using network IB
e07r2n00:23101:26354 [1] NCCL INFO Using network IB
e07r2n17:31184:1572 [0] NCCL INFO Using network IB
e07r2n15:28145:31330 [0] NCCL INFO Using network IB
e07r3n00:24490:27776 [0] NCCL INFO Using network IB
e07r3n01:6047:9752 [1] NCCL INFO Using network IB
e07r2n04:1827:5193 [1] NCCL INFO Using network IB
e07r2n10:29371:32191 [1] NCCL INFO Using network IB
e07r2n19:2109:5605 [1] NCCL INFO Using network IB
e07r2n14:22275:25517 [0] NCCL INFO Using network IB
e07r2n16:26996:30258 [1] NCCL INFO Using network IB
e07r2n11:16528:19774 [2] NCCL INFO Using network IB
e07r2n09:23025:26302 [2] NCCL INFO Using network IB
e07r2n08:30378:1329 [1] NCCL INFO Using network IB
e07r2n18:25918:28644 [1] NCCL INFO Using network IB
e07r2n03:21864:25088 [1] NCCL INFO Using network IB
e07r2n07:11607:15053 [0] NCCL INFO Using network IB
e07r2n13:31210:1485 [3] NCCL INFO Using network IB
e07r1n19:30600:1040 [1] NCCL INFO Using network IB
e07r2n06:26609:29335 [1] NCCL INFO Using network IB
e07r2n00:23099:26356 [0] NCCL INFO Using network IB
e07r2n17:31185:1575 [1] NCCL INFO Using network IB
e07r2n15:28147:31332 [1] NCCL INFO Using network IB
e07r3n00:24491:27778 [1] NCCL INFO Using network IB
e07r2n14:22276:25518 [1] NCCL INFO Using network IB
e07r2n11:16526:19775 [0] NCCL INFO Using network IB
e07r2n09:23024:26305 [1] NCCL INFO Using network IB
e07r2n03:21863:25090 [0] NCCL INFO Using network IB
e07r2n07:11608:15052 [1] NCCL INFO Using network IB
e07r2n13:31206:1486 [0] NCCL INFO Using network IB
e07r2n11:16527:19776 [1] NCCL INFO Using network IB
e07r2n09:23023:26308 [0] NCCL INFO Using network IB
e07r2n13:31209:1487 [2] NCCL INFO Using network IB
e07r2n13:31207:1488 [1] NCCL INFO Using network IB
e07r2n09:23024:26305 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n11:16529:19772 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n19:2107:5602 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n07:11609:15048 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n08:30377:1327 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n08:30378:1329 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r3n00:24493:27774 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n02:4756:8318 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r1n18:15185:19054 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n05:907:4286 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r1n19:30599:1037 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n08:30380:1325 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n03:21864:25088 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n03:21866:25086 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n06:26608:29334 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n11:16527:19776 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n09:23025:26302 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n07:11610:15050 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n09:23026:26303 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n00:23099:26356 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n09:23023:26308 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n00:23102:26350 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n07:11608:15052 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n19:2109:5605 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n06:26609:29335 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n06:26611:29331 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n11:16528:19774 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n17:31185:1575 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n10:29369:32190 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n11:16526:19775 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n10:29371:32191 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n12:16035:18767 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n18:25920:28641 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n14:22275:25517 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n19:2111:5601 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n17:31186:1568 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n19:2110:5599 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n10:29372:32186 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n17:31187:1570 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n12:16037:18770 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n07:11607:15053 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n10:29373:32188 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n18:25919:28639 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n13:31207:1488 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n08:30379:1323 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n02:4758:8319 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r3n00:24490:27776 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n12:16039:18766 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n17:31184:1572 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n14:22276:25518 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n12:16038:18764 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n06:26610:29330 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n00:23103:26351 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n14:22278:25514 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n02:4760:8316 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n18:25917:28643 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n02:4759:8314 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n18:25918:28644 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r1n19:30600:1040 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n03:21863:25090 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r1n18:15187:19060 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n03:21865:25084 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n14:22277:25513 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r3n00:24491:27778 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n01:629:4327 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n16:26998:30253 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r1n18:15189:19057 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r3n00:24492:27773 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n16:26996:30258 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r1n19:30601:1035 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n04:1827:5193 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n05:906:4284 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r1n19:30602:1034 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r3n01:6046:9750 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r3n01:6049:9748 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n05:909:4280 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r3n01:6047:9752 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n05:908:4281 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n13:31210:1485 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r3n01:6048:9746 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r1n18:15188:19056 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n16:26995:30257 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n16:26997:30254 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n15:28145:31330 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n13:31206:1486 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n13:31209:1487 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n15:28147:31332 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n15:28150:31327 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n15:28148:31326 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n04:1829:5188 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e07r2n00:23101:26354 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n04:1828:5186 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n04:1826:5190 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n01:627:4330 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e07r2n01:628:4325 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e07r2n01:626:4329 [0] NCCL INFO Setting affinity for GPU 0 to ff
e07r2n13:31207:1488 [1] NCCL INFO Trees [0] 62/-1/-1->61->60 [1] 62/76/-1->61->60 comm 0x2aedb4000ab0 nRanks 96 busId 26000
e07r2n13:31209:1487 [2] NCCL INFO Trees [0] 63/-1/-1->62->61 [1] 63/-1/-1->62->61 comm 0x2aba34000ab0 nRanks 96 busId 43000
e07r2n13:31210:1485 [3] NCCL INFO Trees [0] -1/-1/-1->63->62 [1] -1/-1/-1->63->62 comm 0x2afab0000ab0 nRanks 96 busId 63000
e07r2n14:22278:25514 [3] NCCL INFO Trees [0] -1/-1/-1->67->66 [1] -1/-1/-1->67->66 comm 0x2b1220000ab0 nRanks 96 busId 63000
e07r2n14:22277:25513 [2] NCCL INFO Trees [0] 67/-1/-1->66->65 [1] 67/-1/-1->66->65 comm 0x2b75dc000ab0 nRanks 96 busId 43000
e07r2n14:22275:25517 [0] NCCL INFO Trees [0] 65/80/-1->64->0 [1] 65/-1/-1->64->68 comm 0x2b6bd8000ab0 nRanks 96 busId 4000
e07r2n15:28145:31330 [0] NCCL INFO Trees [0] 69/-1/-1->68->73 [1] 69/64/-1->68->76 comm 0x2b20a4000ab0 nRanks 96 busId 4000
e07r2n14:22276:25518 [1] NCCL INFO Trees [0] 66/32/-1->65->64 [1] 66/-1/-1->65->64 comm 0x2af10c000ab0 nRanks 96 busId 26000
e07r2n15:28147:31332 [1] NCCL INFO Trees [0] 70/-1/-1->69->68 [1] 70/72/-1->69->68 comm 0x2b1c40000ab0 nRanks 96 busId 26000
e07r2n15:28148:31326 [2] NCCL INFO Trees [0] 71/-1/-1->70->69 [1] 71/-1/-1->70->69 comm 0x2afcc0000ab0 nRanks 96 busId 43000
e07r2n15:28150:31327 [3] NCCL INFO Trees [0] -1/-1/-1->71->70 [1] -1/-1/-1->71->70 comm 0x2b3cb0000ab0 nRanks 96 busId 63000
e07r2n19:2109:5605 [1] NCCL INFO Trees [0] 86/-1/-1->85->84 [1] 86/88/-1->85->84 comm 0x2b3080000ab0 nRanks 96 busId 26000
e07r2n18:25920:28641 [3] NCCL INFO Trees [0] -1/-1/-1->83->82 [1] -1/-1/-1->83->82 comm 0x2af828000ab0 nRanks 96 busId 63000
e07r2n17:31187:1570 [3] NCCL INFO Trees [0] -1/-1/-1->79->78 [1] -1/-1/-1->79->78 comm 0x2b137c000ab0 nRanks 96 busId 63000
e07r2n19:2111:5601 [3] NCCL INFO Trees [0] -1/-1/-1->87->86 [1] -1/-1/-1->87->86 comm 0x2b3334000ab0 nRanks 96 busId 63000
e07r2n19:2110:5599 [2] NCCL INFO Trees [0] 87/-1/-1->86->85 [1] 87/-1/-1->86->85 comm 0x2b9f58000ab0 nRanks 96 busId 43000
e07r2n17:31184:1572 [0] NCCL INFO Trees [0] 77/-1/-1->76->72 [1] 77/68/-1->76->61 comm 0x2bb19c000ab0 nRanks 96 busId 4000
e07r2n19:2107:5602 [0] NCCL INFO Trees [0] 85/-1/-1->84->89 [1] 85/80/-1->84->77 comm 0x2b5ad0000ab0 nRanks 96 busId 4000
e07r2n03:21866:25086 [3] NCCL INFO Trees [0] -1/-1/-1->23->22 [1] -1/-1/-1->23->22 comm 0x2b6e40000ab0 nRanks 96 busId 63000
e07r2n11:16529:19772 [3] NCCL INFO Trees [0] -1/-1/-1->55->54 [1] -1/-1/-1->55->54 comm 0x2b84d8000ab0 nRanks 96 busId 63000
e07r2n17:31186:1568 [2] NCCL INFO Trees [0] 79/-1/-1->78->77 [1] 79/-1/-1->78->77 comm 0x2b30ec000ab0 nRanks 96 busId 43000
e07r2n16:26995:30257 [0] NCCL INFO Trees [0] 73/76/-1->72->81 [1] 73/-1/-1->72->69 comm 0x2b7ef4000ab0 nRanks 96 busId 4000
e07r2n12:16039:18766 [3] NCCL INFO Trees [0] -1/-1/-1->59->58 [1] -1/-1/-1->59->58 comm 0x2b4134000ab0 nRanks 96 busId 63000
e07r2n18:25917:28643 [0] NCCL INFO Trees [0] 81/88/-1->80->64 [1] 81/-1/-1->80->84 comm 0x2b97a0000ab0 nRanks 96 busId 4000
e07r3n01:6046:9750 [0] NCCL INFO Trees [0] 93/-1/-1->92->88 [1] 93/28/-1->92->-1 comm 0x2b0dd4000ab0 nRanks 96 busId 4000
e07r2n17:31185:1575 [1] NCCL INFO Trees [0] 78/-1/-1->77->76 [1] 78/84/-1->77->76 comm 0x2ac95c000ab0 nRanks 96 busId 26000
e07r2n16:26996:30258 [1] NCCL INFO Trees [0] 74/68/-1->73->72 [1] 74/-1/-1->73->72 comm 0x2ae0fc000ab0 nRanks 96 busId 26000
e07r2n12:16037:18770 [1] NCCL INFO Trees [0] 58/52/-1->57->56 [1] 58/-1/-1->57->56 comm 0x2ae364000ab0 nRanks 96 busId 26000
e07r2n18:25918:28644 [1] NCCL INFO Trees [0] 82/72/-1->81->80 [1] 82/-1/-1->81->80 comm 0x2b0bb8000ab0 nRanks 96 busId 26000
e07r2n13:31206:1486 [0] NCCL INFO Trees [0] 61/-1/-1->60->56 [1] 61/44/-1->60->29 comm 0x2aec38000ab0 nRanks 96 busId 4000
e07r2n06:26610:29330 [2] NCCL INFO Trees [0] 35/-1/-1->34->33 [1] 35/-1/-1->34->33 comm 0x2b9594000ab0 nRanks 96 busId 43000
e07r1n18:15187:19060 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 comm 0x2ac9ac000ab0 nRanks 96 busId 26000
e07r1n18:15185:19054 [0] NCCL INFO Channel 00/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
e07r1n18:15185:19054 [0] NCCL INFO Channel 01/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
e07r1n18:15185:19054 [0] NCCL INFO Trees [0] 1/64/-1->0->-1 [1] 1/-1/-1->0->4 comm 0x2b2ef8000ab0 nRanks 96 busId 4000
e07r2n16:26997:30254 [2] NCCL INFO Trees [0] 75/-1/-1->74->73 [1] 75/-1/-1->74->73 comm 0x2b23d4000ab0 nRanks 96 busId 43000
e07r2n12:16035:18767 [0] NCCL INFO Trees [0] 57/60/-1->56->48 [1] 57/-1/-1->56->53 comm 0x2b858c000ab0 nRanks 96 busId 4000
e07r2n18:25919:28639 [2] NCCL INFO Trees [0] 83/-1/-1->82->81 [1] 83/-1/-1->82->81 comm 0x2af1a8000ab0 nRanks 96 busId 43000
e07r2n06:26608:29334 [0] NCCL INFO Trees [0] 33/48/-1->32->65 [1] 33/-1/-1->32->36 comm 0x2b2c04000ab0 nRanks 96 busId 4000
e07r2n05:909:4280 [3] NCCL INFO Trees [0] -1/-1/-1->31->30 [1] -1/-1/-1->31->30 comm 0x2ac550000ab0 nRanks 96 busId 63000
e07r2n00:23103:26351 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] -1/-1/-1->11->10 comm 0x2b8604000ab0 nRanks 96 busId 63000
e07r3n00:24490:27776 [0] NCCL INFO Trees [0] 89/92/-1->88->80 [1] 89/-1/-1->88->85 comm 0x2b876c000ab0 nRanks 96 busId 4000
e07r2n01:627:4330 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/20/-1->13->12 comm 0x2b7854000ab0 nRanks 96 busId 26000
e07r2n03:21865:25084 [2] NCCL INFO Trees [0] 23/-1/-1->22->21 [1] 23/-1/-1->22->21 comm 0x2b628c000ab0 nRanks 96 busId 43000
e07r1n18:15189:19057 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 comm 0x2b6fbc000ab0 nRanks 96 busId 63000
e07r1n18:15188:19056 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 comm 0x2b6c7c000ab0 nRanks 96 busId 43000
e07r2n16:26998:30253 [3] NCCL INFO Trees [0] -1/-1/-1->75->74 [1] -1/-1/-1->75->74 comm 0x2b5f04000ab0 nRanks 96 busId 63000
e07r2n12:16038:18764 [2] NCCL INFO Trees [0] 59/-1/-1->58->57 [1] 59/-1/-1->58->57 comm 0x2b9664000ab0 nRanks 96 busId 43000
e07r2n06:26609:29335 [1] NCCL INFO Trees [0] 34/16/-1->33->32 [1] 34/-1/-1->33->32 comm 0x2ae97c000ab0 nRanks 96 busId 26000
e07r2n05:908:4281 [2] NCCL INFO Trees [0] 31/-1/-1->30->29 [1] 31/-1/-1->30->29 comm 0x2b03d0000ab0 nRanks 96 busId 43000
e07r2n00:23102:26350 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 comm 0x2ace64000ab0 nRanks 96 busId 43000
e07r3n00:24491:27778 [1] NCCL INFO Trees [0] 90/84/-1->89->88 [1] 90/-1/-1->89->88 comm 0x2b9dcc000ab0 nRanks 96 busId 26000
e07r2n01:626:4329 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] 13/4/-1->12->28 comm 0x2ae9b8000ab0 nRanks 96 busId 4000
e07r2n03:21863:25090 [0] NCCL INFO Trees [0] 21/-1/-1->20->25 [1] 21/16/-1->20->13 comm 0x2b3d8c000ab0 nRanks 96 busId 4000
e07r2n11:16528:19774 [2] NCCL INFO Trees [0] 55/-1/-1->54->53 [1] 55/-1/-1->54->53 comm 0x2b8cf0000ab0 nRanks 96 busId 43000
e07r3n01:6049:9748 [3] NCCL INFO Trees [0] -1/-1/-1->95->94 [1] -1/-1/-1->95->94 comm 0x2b508c000ab0 nRanks 96 busId 63000
e07r2n04:1829:5188 [3] NCCL INFO Trees [0] -1/-1/-1->27->26 [1] -1/-1/-1->27->26 comm 0x2b68a8000ab0 nRanks 96 busId 63000
e07r2n09:23025:26302 [2] NCCL INFO Trees [0] 47/-1/-1->46->45 [1] 47/-1/-1->46->45 comm 0x2b4748000ab0 nRanks 96 busId 43000
e07r2n10:29372:32186 [2] NCCL INFO Trees [0] 51/-1/-1->50->49 [1] 51/-1/-1->50->49 comm 0x2b4970000ab0 nRanks 96 busId 43000
e07r2n08:30380:1325 [3] NCCL INFO Trees [0] -1/-1/-1->43->42 [1] -1/-1/-1->43->42 comm 0x2b8818000ab0 nRanks 96 busId 63000
e07r2n07:11610:15050 [3] NCCL INFO Trees [0] -1/-1/-1->39->38 [1] -1/-1/-1->39->38 comm 0x2affe4000ab0 nRanks 96 busId 63000
e07r2n06:26611:29331 [3] NCCL INFO Trees [0] -1/-1/-1->35->34 [1] -1/-1/-1->35->34 comm 0x2b62e4000ab0 nRanks 96 busId 63000
e07r2n05:906:4284 [0] NCCL INFO Trees [0] 29/-1/-1->28->24 [1] 29/12/-1->28->92 comm 0x2ba624000ab0 nRanks 96 busId 4000
e07r2n00:23101:26354 [1] NCCL INFO Trees [0] 10/4/-1->9->8 [1] 10/-1/-1->9->8 comm 0x2ac104000ab0 nRanks 96 busId 26000
e07r3n00:24492:27773 [2] NCCL INFO Trees [0] 91/-1/-1->90->89 [1] 91/-1/-1->90->89 comm 0x2b9934000ab0 nRanks 96 busId 43000
e07r2n01:629:4327 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] -1/-1/-1->15->14 comm 0x2ad0f4000ab0 nRanks 96 busId 63000
e07r2n03:21864:25088 [1] NCCL INFO Trees [0] 22/-1/-1->21->20 [1] 22/24/-1->21->20 comm 0x2b34e0000ab0 nRanks 96 busId 26000
e07r2n11:16526:19775 [0] NCCL INFO Trees [0] 53/-1/-1->52->57 [1] 53/48/-1->52->45 comm 0x2b86d4000ab0 nRanks 96 busId 4000
e07r3n01:6048:9746 [2] NCCL INFO Trees [0] 95/-1/-1->94->93 [1] 95/-1/-1->94->93 comm 0x2b4198000ab0 nRanks 96 busId 43000
e07r2n04:1826:5190 [0] NCCL INFO Trees [0] 25/28/-1->24->16 [1] 25/-1/-1->24->21 comm 0x2b0c64000ab0 nRanks 96 busId 4000
e07r2n09:23026:26303 [3] NCCL INFO Trees [0] -1/-1/-1->47->46 [1] -1/-1/-1->47->46 comm 0x2b4b4c000ab0 nRanks 96 busId 63000
e07r2n10:29373:32188 [3] NCCL INFO Trees [0] -1/-1/-1->51->50 [1] -1/-1/-1->51->50 comm 0x2b2da4000ab0 nRanks 96 busId 63000
e07r2n08:30379:1323 [2] NCCL INFO Trees [0] 43/-1/-1->42->41 [1] 43/-1/-1->42->41 comm 0x2adfbc000ab0 nRanks 96 busId 43000
e07r2n07:11609:15048 [2] NCCL INFO Trees [0] 39/-1/-1->38->37 [1] 39/-1/-1->38->37 comm 0x2accec000ab0 nRanks 96 busId 43000
e07r1n19:30600:1040 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/8/-1->5->4 comm 0x2b48a0000ab0 nRanks 96 busId 26000
e07r2n02:4756:8318 [0] NCCL INFO Trees [0] 17/24/-1->16->33 [1] 17/-1/-1->16->20 comm 0x2b71c4000ab0 nRanks 96 busId 4000
e07r2n05:907:4286 [1] NCCL INFO Trees [0] 30/-1/-1->29->28 [1] 30/60/-1->29->28 comm 0x2af92c000ab0 nRanks 96 busId 26000
e07r2n00:23099:26356 [0] NCCL INFO Trees [0] 9/12/-1->8->17 [1] 9/-1/-1->8->5 comm 0x2b7e4c000ab0 nRanks 96 busId 4000
e07r3n00:24493:27774 [3] NCCL INFO Trees [0] -1/-1/-1->91->90 [1] -1/-1/-1->91->90 comm 0x2b0888000ab0 nRanks 96 busId 63000
e07r2n01:628:4325 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 comm 0x2aeb74000ab0 nRanks 96 busId 43000
e07r2n11:16527:19776 [1] NCCL INFO Trees [0] 54/-1/-1->53->52 [1] 54/56/-1->53->52 comm 0x2aebfc000ab0 nRanks 96 busId 26000
e07r3n01:6047:9752 [1] NCCL INFO Trees [0] 94/-1/-1->93->92 [1] 94/-1/-1->93->92 comm 0x2ade98000ab0 nRanks 96 busId 26000
e07r2n04:1827:5193 [1] NCCL INFO Trees [0] 26/20/-1->25->24 [1] 26/-1/-1->25->24 comm 0x2ac224000ab0 nRanks 96 busId 26000
e07r2n09:23023:26308 [0] NCCL INFO Trees [0] 45/-1/-1->44->40 [1] 45/36/-1->44->60 comm 0x2af9d0000ab0 nRanks 96 busId 4000
e07r2n10:29371:32191 [1] NCCL INFO Trees [0] 50/40/-1->49->48 [1] 50/-1/-1->49->48 comm 0x2ad734000ab0 nRanks 96 busId 26000
e07r2n08:30378:1329 [1] NCCL INFO Trees [0] 42/36/-1->41->40 [1] 42/-1/-1->41->40 comm 0x2b2e18000ab0 nRanks 96 busId 26000
e07r2n07:11607:15053 [0] NCCL INFO Trees [0] 37/-1/-1->36->41 [1] 37/32/-1->36->44 comm 0x2ac9e0000ab0 nRanks 96 busId 4000
e07r1n19:30599:1037 [0] NCCL INFO Trees [0] 5/-1/-1->4->9 [1] 5/0/-1->4->12 comm 0x2b7088000ab0 nRanks 96 busId 4000
e07r2n02:4758:8319 [1] NCCL INFO Trees [0] 18/8/-1->17->16 [1] 18/-1/-1->17->16 comm 0x2b8694000ab0 nRanks 96 busId 26000
e07r2n04:1828:5186 [2] NCCL INFO Trees [0] 27/-1/-1->26->25 [1] 27/-1/-1->26->25 comm 0x2b227c000ab0 nRanks 96 busId 43000
e07r2n09:23024:26305 [1] NCCL INFO Trees [0] 46/-1/-1->45->44 [1] 46/52/-1->45->44 comm 0x2b3c5c000ab0 nRanks 96 busId 26000
e07r2n10:29369:32190 [0] NCCL INFO Trees [0] 49/56/-1->48->32 [1] 49/-1/-1->48->52 comm 0x2b2760000ab0 nRanks 96 busId 4000
e07r2n08:30377:1327 [0] NCCL INFO Trees [0] 41/44/-1->40->49 [1] 41/-1/-1->40->37 comm 0x2b4274000ab0 nRanks 96 busId 4000
e07r2n07:11608:15052 [1] NCCL INFO Trees [0] 38/-1/-1->37->36 [1] 38/40/-1->37->36 comm 0x2aed18000ab0 nRanks 96 busId 26000
e07r1n19:30601:1035 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 comm 0x2b9774000ab0 nRanks 96 busId 43000
e07r2n02:4759:8314 [2] NCCL INFO Trees [0] 19/-1/-1->18->17 [1] 19/-1/-1->18->17 comm 0x2b3524000ab0 nRanks 96 busId 43000
e07r1n19:30602:1034 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 comm 0x2b2e08000ab0 nRanks 96 busId 63000
e07r2n02:4760:8316 [3] NCCL INFO Trees [0] -1/-1/-1->19->18 [1] -1/-1/-1->19->18 comm 0x2b7558000ab0 nRanks 96 busId 63000
e07r2n13:31209:1487 [2] NCCL INFO Channel 00 : 62[43000] -> 63[63000] via P2P/IPC comm 0x2aba34000ab0 nRanks 96
e07r2n13:31207:1488 [1] NCCL INFO Channel 00 : 61[26000] -> 62[43000] via P2P/IPC comm 0x2aedb4000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 00/0 : 67[63000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b20a4000ab0 nRanks 96
e07r2n14:22275:25517 [0] NCCL INFO Channel 00/0 : 63[63000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b6bd8000ab0 nRanks 96
e07r2n13:31209:1487 [2] NCCL INFO Channel 01 : 62[43000] -> 63[63000] via P2P/IPC comm 0x2aba34000ab0 nRanks 96
e07r2n14:22276:25518 [1] NCCL INFO Channel 00 : 65[26000] -> 66[43000] via P2P/IPC comm 0x2af10c000ab0 nRanks 96
e07r2n14:22277:25513 [2] NCCL INFO Channel 00 : 66[43000] -> 67[63000] via P2P/IPC comm 0x2b75dc000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 00/0 : 75[63000] -> 76[4000] [receive] via NET/IB/0 comm 0x2bb19c000ab0 nRanks 96
e07r2n19:2109:5605 [1] NCCL INFO Channel 00 : 85[26000] -> 86[43000] via P2P/IPC comm 0x2b3080000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 00/0 : 79[63000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b97a0000ab0 nRanks 96
e07r3n01:6046:9750 [0] NCCL INFO Channel 00/0 : 91[63000] -> 92[4000] [receive] via NET/IB/0 comm 0x2b0dd4000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 00/0 : 83[63000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5ad0000ab0 nRanks 96
e07r2n13:31207:1488 [1] NCCL INFO Channel 01 : 61[26000] -> 62[43000] via P2P/IPC comm 0x2aedb4000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 00/0 : 71[63000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e07r2n15:28148:31326 [2] NCCL INFO Channel 00 : 70[43000] -> 71[63000] via P2P/IPC comm 0x2afcc0000ab0 nRanks 96
e07r2n19:2110:5599 [2] NCCL INFO Channel 00 : 86[43000] -> 87[63000] via P2P/IPC comm 0x2b9f58000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 00/0 : 59[63000] -> 60[4000] [receive] via NET/IB/0 comm 0x2aec38000ab0 nRanks 96
e07r2n14:22277:25513 [2] NCCL INFO Channel 01 : 66[43000] -> 67[63000] via P2P/IPC comm 0x2b75dc000ab0 nRanks 96
e07r2n19:2109:5605 [1] NCCL INFO Channel 01 : 85[26000] -> 86[43000] via P2P/IPC comm 0x2b3080000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 00/0 : 55[63000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b858c000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 00/0 : 27[63000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba624000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 00/0 : 23[63000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b0c64000ab0 nRanks 96
e07r2n17:31186:1568 [2] NCCL INFO Channel 00 : 78[43000] -> 79[63000] via P2P/IPC comm 0x2b30ec000ab0 nRanks 96
e07r2n12:16037:18770 [1] NCCL INFO Channel 00 : 57[26000] -> 58[43000] via P2P/IPC comm 0x2ae364000ab0 nRanks 96
e07r2n17:31185:1575 [1] NCCL INFO Channel 00 : 77[26000] -> 78[43000] via P2P/IPC comm 0x2ac95c000ab0 nRanks 96
e07r3n00:24490:27776 [0] NCCL INFO Channel 00/0 : 87[63000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b876c000ab0 nRanks 96
e07r2n16:26996:30258 [1] NCCL INFO Channel 00 : 73[26000] -> 74[43000] via P2P/IPC comm 0x2ae0fc000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 00/0 : 51[63000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b86d4000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 00/0 : 31[63000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2c04000ab0 nRanks 96
e07r2n18:25919:28639 [2] NCCL INFO Channel 00 : 82[43000] -> 83[63000] via P2P/IPC comm 0x2af1a8000ab0 nRanks 96
e07r2n15:28148:31326 [2] NCCL INFO Channel 01 : 70[43000] -> 71[63000] via P2P/IPC comm 0x2afcc0000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 00/0 : 11[63000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae9b8000ab0 nRanks 96
e07r1n18:15187:19060 [1] NCCL INFO Channel 00 : 1[26000] -> 2[43000] via P2P/IPC comm 0x2ac9ac000ab0 nRanks 96
e07r2n12:16038:18764 [2] NCCL INFO Channel 00 : 58[43000] -> 59[63000] via P2P/IPC comm 0x2b9664000ab0 nRanks 96
e07r3n00:24491:27778 [1] NCCL INFO Channel 00 : 89[26000] -> 90[43000] via P2P/IPC comm 0x2b9dcc000ab0 nRanks 96
e07r2n16:26997:30254 [2] NCCL INFO Channel 00 : 74[43000] -> 75[63000] via P2P/IPC comm 0x2b23d4000ab0 nRanks 96
e07r2n06:26610:29330 [2] NCCL INFO Channel 00 : 34[43000] -> 35[63000] via P2P/IPC comm 0x2b9594000ab0 nRanks 96
e07r2n18:25918:28644 [1] NCCL INFO Channel 00 : 81[26000] -> 82[43000] via P2P/IPC comm 0x2b0bb8000ab0 nRanks 96
e07r2n01:627:4330 [1] NCCL INFO Channel 00 : 13[26000] -> 14[43000] via P2P/IPC comm 0x2b7854000ab0 nRanks 96
e07r2n19:2110:5599 [2] NCCL INFO Channel 01 : 86[43000] -> 87[63000] via P2P/IPC comm 0x2b9f58000ab0 nRanks 96
e07r2n14:22276:25518 [1] NCCL INFO Channel 01 : 65[26000] -> 66[43000] via P2P/IPC comm 0x2af10c000ab0 nRanks 96
e07r3n01:6048:9746 [2] NCCL INFO Channel 00 : 94[43000] -> 95[63000] via P2P/IPC comm 0x2b4198000ab0 nRanks 96
e07r2n00:23102:26350 [2] NCCL INFO Channel 00 : 10[43000] -> 11[63000] via P2P/IPC comm 0x2ace64000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 00/0 : 19[63000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3d8c000ab0 nRanks 96
e07r2n05:908:4281 [2] NCCL INFO Channel 00 : 30[43000] -> 31[63000] via P2P/IPC comm 0x2b03d0000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 00/0 : 43[63000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af9d0000ab0 nRanks 96
e07r2n04:1828:5186 [2] NCCL INFO Channel 00 : 26[43000] -> 27[63000] via P2P/IPC comm 0x2b227c000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 00/0 : 47[63000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b2760000ab0 nRanks 96
e07r2n08:30379:1323 [2] NCCL INFO Channel 00 : 42[43000] -> 43[63000] via P2P/IPC comm 0x2adfbc000ab0 nRanks 96
e07r1n18:15185:19054 [0] NCCL INFO Channel 00/0 : 95[63000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b2ef8000ab0 nRanks 96
e07r1n18:15187:19060 [1] NCCL INFO Channel 01 : 1[26000] -> 2[43000] via P2P/IPC comm 0x2ac9ac000ab0 nRanks 96
e07r2n12:16038:18764 [2] NCCL INFO Channel 01 : 58[43000] -> 59[63000] via P2P/IPC comm 0x2b9664000ab0 nRanks 96
e07r3n00:24492:27773 [2] NCCL INFO Channel 00 : 90[43000] -> 91[63000] via P2P/IPC comm 0x2b9934000ab0 nRanks 96
e07r2n16:26996:30258 [1] NCCL INFO Channel 01 : 73[26000] -> 74[43000] via P2P/IPC comm 0x2ae0fc000ab0 nRanks 96
e07r2n06:26609:29335 [1] NCCL INFO Channel 00 : 33[26000] -> 34[43000] via P2P/IPC comm 0x2ae97c000ab0 nRanks 96
e07r2n18:25919:28639 [2] NCCL INFO Channel 01 : 82[43000] -> 83[63000] via P2P/IPC comm 0x2af1a8000ab0 nRanks 96
e07r2n01:628:4325 [2] NCCL INFO Channel 00 : 14[43000] -> 15[63000] via P2P/IPC comm 0x2aeb74000ab0 nRanks 96
e07r3n01:6047:9752 [1] NCCL INFO Channel 00 : 93[26000] -> 94[43000] via P2P/IPC comm 0x2ade98000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 00/0 : 7[63000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7e4c000ab0 nRanks 96
e07r2n03:21865:25084 [2] NCCL INFO Channel 00 : 22[43000] -> 23[63000] via P2P/IPC comm 0x2b628c000ab0 nRanks 96
e07r2n05:907:4286 [1] NCCL INFO Channel 00 : 29[26000] -> 30[43000] via P2P/IPC comm 0x2af92c000ab0 nRanks 96
e07r2n09:23025:26302 [2] NCCL INFO Channel 00 : 46[43000] -> 47[63000] via P2P/IPC comm 0x2b4748000ab0 nRanks 96
e07r2n04:1827:5193 [1] NCCL INFO Channel 00 : 25[26000] -> 26[43000] via P2P/IPC comm 0x2ac224000ab0 nRanks 96
e07r2n10:29372:32186 [2] NCCL INFO Channel 00 : 50[43000] -> 51[63000] via P2P/IPC comm 0x2b4970000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 00/0 : 39[63000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4274000ab0 nRanks 96
e07r2n17:31185:1575 [1] NCCL INFO Channel 01 : 77[26000] -> 78[43000] via P2P/IPC comm 0x2ac95c000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 00/0 : 35[63000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac9e0000ab0 nRanks 96
e07r2n11:16528:19774 [2] NCCL INFO Channel 00 : 54[43000] -> 55[63000] via P2P/IPC comm 0x2b8cf0000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 00/0 : 15[63000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b71c4000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 00/0 : 3[63000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b7088000ab0 nRanks 96
e07r1n18:15188:19056 [2] NCCL INFO Channel 00 : 2[43000] -> 3[63000] via P2P/IPC comm 0x2b6c7c000ab0 nRanks 96
e07r1n18:15188:19056 [2] NCCL INFO Channel 01 : 2[43000] -> 3[63000] via P2P/IPC comm 0x2b6c7c000ab0 nRanks 96
e07r2n12:16037:18770 [1] NCCL INFO Channel 01 : 57[26000] -> 58[43000] via P2P/IPC comm 0x2ae364000ab0 nRanks 96
e07r3n00:24491:27778 [1] NCCL INFO Channel 01 : 89[26000] -> 90[43000] via P2P/IPC comm 0x2b9dcc000ab0 nRanks 96
e07r2n16:26997:30254 [2] NCCL INFO Channel 01 : 74[43000] -> 75[63000] via P2P/IPC comm 0x2b23d4000ab0 nRanks 96
e07r2n06:26609:29335 [1] NCCL INFO Channel 01 : 33[26000] -> 34[43000] via P2P/IPC comm 0x2ae97c000ab0 nRanks 96
e07r2n18:25918:28644 [1] NCCL INFO Channel 01 : 81[26000] -> 82[43000] via P2P/IPC comm 0x2b0bb8000ab0 nRanks 96
e07r2n01:627:4330 [1] NCCL INFO Channel 01 : 13[26000] -> 14[43000] via P2P/IPC comm 0x2b7854000ab0 nRanks 96
e07r3n01:6048:9746 [2] NCCL INFO Channel 01 : 94[43000] -> 95[63000] via P2P/IPC comm 0x2b4198000ab0 nRanks 96
e07r2n00:23102:26350 [2] NCCL INFO Channel 01 : 10[43000] -> 11[63000] via P2P/IPC comm 0x2ace64000ab0 nRanks 96
e07r2n03:21865:25084 [2] NCCL INFO Channel 01 : 22[43000] -> 23[63000] via P2P/IPC comm 0x2b628c000ab0 nRanks 96
e07r2n05:908:4281 [2] NCCL INFO Channel 01 : 30[43000] -> 31[63000] via P2P/IPC comm 0x2b03d0000ab0 nRanks 96
e07r2n09:23024:26305 [1] NCCL INFO Channel 00 : 45[26000] -> 46[43000] via P2P/IPC comm 0x2b3c5c000ab0 nRanks 96
e07r2n04:1828:5186 [2] NCCL INFO Channel 01 : 26[43000] -> 27[63000] via P2P/IPC comm 0x2b227c000ab0 nRanks 96
e07r2n10:29371:32191 [1] NCCL INFO Channel 00 : 49[26000] -> 50[43000] via P2P/IPC comm 0x2ad734000ab0 nRanks 96
e07r2n08:30378:1329 [1] NCCL INFO Channel 00 : 41[26000] -> 42[43000] via P2P/IPC comm 0x2b2e18000ab0 nRanks 96
e07r2n17:31186:1568 [2] NCCL INFO Channel 01 : 78[43000] -> 79[63000] via P2P/IPC comm 0x2b30ec000ab0 nRanks 96
e07r2n07:11608:15052 [1] NCCL INFO Channel 00 : 37[26000] -> 38[43000] via P2P/IPC comm 0x2aed18000ab0 nRanks 96
e07r2n11:16527:19776 [1] NCCL INFO Channel 00 : 53[26000] -> 54[43000] via P2P/IPC comm 0x2aebfc000ab0 nRanks 96
e07r2n02:4759:8314 [2] NCCL INFO Channel 00 : 18[43000] -> 19[63000] via P2P/IPC comm 0x2b3524000ab0 nRanks 96
e07r1n19:30600:1040 [1] NCCL INFO Channel 00 : 5[26000] -> 6[43000] via P2P/IPC comm 0x2b48a0000ab0 nRanks 96
e07r3n00:24492:27773 [2] NCCL INFO Channel 01 : 90[43000] -> 91[63000] via P2P/IPC comm 0x2b9934000ab0 nRanks 96
e07r2n06:26610:29330 [2] NCCL INFO Channel 01 : 34[43000] -> 35[63000] via P2P/IPC comm 0x2b9594000ab0 nRanks 96
e07r2n01:628:4325 [2] NCCL INFO Channel 01 : 14[43000] -> 15[63000] via P2P/IPC comm 0x2aeb74000ab0 nRanks 96
e07r3n01:6047:9752 [1] NCCL INFO Channel 01 : 93[26000] -> 94[43000] via P2P/IPC comm 0x2ade98000ab0 nRanks 96
e07r2n00:23101:26354 [1] NCCL INFO Channel 00 : 9[26000] -> 10[43000] via P2P/IPC comm 0x2ac104000ab0 nRanks 96
e07r2n03:21864:25088 [1] NCCL INFO Channel 00 : 21[26000] -> 22[43000] via P2P/IPC comm 0x2b34e0000ab0 nRanks 96
e07r2n05:907:4286 [1] NCCL INFO Channel 01 : 29[26000] -> 30[43000] via P2P/IPC comm 0x2af92c000ab0 nRanks 96
e07r2n09:23025:26302 [2] NCCL INFO Channel 01 : 46[43000] -> 47[63000] via P2P/IPC comm 0x2b4748000ab0 nRanks 96
e07r2n10:29372:32186 [2] NCCL INFO Channel 01 : 50[43000] -> 51[63000] via P2P/IPC comm 0x2b4970000ab0 nRanks 96
e07r2n08:30379:1323 [2] NCCL INFO Channel 01 : 42[43000] -> 43[63000] via P2P/IPC comm 0x2adfbc000ab0 nRanks 96
e07r2n07:11609:15048 [2] NCCL INFO Channel 00 : 38[43000] -> 39[63000] via P2P/IPC comm 0x2accec000ab0 nRanks 96
e07r2n11:16528:19774 [2] NCCL INFO Channel 01 : 54[43000] -> 55[63000] via P2P/IPC comm 0x2b8cf0000ab0 nRanks 96
e07r2n02:4758:8319 [1] NCCL INFO Channel 00 : 17[26000] -> 18[43000] via P2P/IPC comm 0x2b8694000ab0 nRanks 96
e07r1n19:30601:1035 [2] NCCL INFO Channel 00 : 6[43000] -> 7[63000] via P2P/IPC comm 0x2b9774000ab0 nRanks 96
e07r2n00:23101:26354 [1] NCCL INFO Channel 01 : 9[26000] -> 10[43000] via P2P/IPC comm 0x2ac104000ab0 nRanks 96
e07r2n03:21864:25088 [1] NCCL INFO Channel 01 : 21[26000] -> 22[43000] via P2P/IPC comm 0x2b34e0000ab0 nRanks 96
e07r2n09:23024:26305 [1] NCCL INFO Channel 01 : 45[26000] -> 46[43000] via P2P/IPC comm 0x2b3c5c000ab0 nRanks 96
e07r2n10:29371:32191 [1] NCCL INFO Channel 01 : 49[26000] -> 50[43000] via P2P/IPC comm 0x2ad734000ab0 nRanks 96
e07r2n08:30378:1329 [1] NCCL INFO Channel 01 : 41[26000] -> 42[43000] via P2P/IPC comm 0x2b2e18000ab0 nRanks 96
e07r2n07:11608:15052 [1] NCCL INFO Channel 01 : 37[26000] -> 38[43000] via P2P/IPC comm 0x2aed18000ab0 nRanks 96
e07r2n11:16527:19776 [1] NCCL INFO Channel 01 : 53[26000] -> 54[43000] via P2P/IPC comm 0x2aebfc000ab0 nRanks 96
e07r2n02:4758:8319 [1] NCCL INFO Channel 01 : 17[26000] -> 18[43000] via P2P/IPC comm 0x2b8694000ab0 nRanks 96
e07r1n19:30600:1040 [1] NCCL INFO Channel 01 : 5[26000] -> 6[43000] via P2P/IPC comm 0x2b48a0000ab0 nRanks 96
e07r2n07:11609:15048 [2] NCCL INFO Channel 01 : 38[43000] -> 39[63000] via P2P/IPC comm 0x2accec000ab0 nRanks 96
e07r2n02:4759:8314 [2] NCCL INFO Channel 01 : 18[43000] -> 19[63000] via P2P/IPC comm 0x2b3524000ab0 nRanks 96
e07r1n19:30601:1035 [2] NCCL INFO Channel 01 : 6[43000] -> 7[63000] via P2P/IPC comm 0x2b9774000ab0 nRanks 96
e07r2n04:1827:5193 [1] NCCL INFO Channel 01 : 25[26000] -> 26[43000] via P2P/IPC comm 0x2ac224000ab0 nRanks 96
e07r2n15:28147:31332 [1] NCCL INFO Channel 00 : 69[26000] -> 70[43000] via P2P/IPC comm 0x2b1c40000ab0 nRanks 96
e07r2n15:28147:31332 [1] NCCL INFO Channel 01 : 69[26000] -> 70[43000] via P2P/IPC comm 0x2b1c40000ab0 nRanks 96
e07r2n13:31210:1485 [3] NCCL INFO Channel 00/0 : 63[63000] -> 64[4000] [send] via NET/IB/0 comm 0x2afab0000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 01/0 : 67[63000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b20a4000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 00 : 68[4000] -> 69[26000] via P2P/IPC comm 0x2b20a4000ab0 nRanks 96
e07r2n19:2111:5601 [3] NCCL INFO Channel 00/0 : 87[63000] -> 88[4000] [send] via NET/IB/0 comm 0x2b3334000ab0 nRanks 96
e07r3n01:6049:9748 [3] NCCL INFO Channel 00/0 : 95[63000] -> 0[4000] [send] via NET/IB/0 comm 0x2b508c000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 01/0 : 83[63000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5ad0000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 01/0 : 27[63000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba624000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 00 : 28[4000] -> 29[26000] via P2P/IPC comm 0x2ba624000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 00 : 84[4000] -> 85[26000] via P2P/IPC comm 0x2b5ad0000ab0 nRanks 96
e07r3n00:24490:27776 [0] NCCL INFO Channel 01/0 : 87[63000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b876c000ab0 nRanks 96
e07r3n01:6046:9750 [0] NCCL INFO Channel 01/0 : 91[63000] -> 92[4000] [receive] via NET/IB/0 comm 0x2b0dd4000ab0 nRanks 96
e07r3n01:6046:9750 [0] NCCL INFO Channel 00 : 92[4000] -> 93[26000] via P2P/IPC comm 0x2b0dd4000ab0 nRanks 96
e07r3n00:24490:27776 [0] NCCL INFO Channel 00 : 88[4000] -> 89[26000] via P2P/IPC comm 0x2b876c000ab0 nRanks 96
e07r2n18:25920:28641 [3] NCCL INFO Channel 00/0 : 83[63000] -> 84[4000] [send] via NET/IB/0 comm 0x2af828000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 01/0 : 79[63000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b97a0000ab0 nRanks 96
e07r2n04:1829:5188 [3] NCCL INFO Channel 00/0 : 27[63000] -> 28[4000] [send] via NET/IB/0 comm 0x2b68a8000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 01 : 68[4000] -> 69[26000] via P2P/IPC comm 0x2b20a4000ab0 nRanks 96
e07r2n14:22275:25517 [0] NCCL INFO Channel 01/0 : 63[63000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b6bd8000ab0 nRanks 96
e07r2n14:22275:25517 [0] NCCL INFO Channel 00 : 64[4000] -> 65[26000] via P2P/IPC comm 0x2b6bd8000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 00 : 80[4000] -> 81[26000] via P2P/IPC comm 0x2b97a0000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 01/0 : 23[63000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b0c64000ab0 nRanks 96
e07r2n14:22278:25514 [3] NCCL INFO Channel 00/0 : 67[63000] -> 68[4000] [send] via NET/IB/0 comm 0x2b1220000ab0 nRanks 96
e07r2n06:26611:29331 [3] NCCL INFO Channel 00/0 : 35[63000] -> 36[4000] [send] via NET/IB/0 comm 0x2b62e4000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 00 : 24[4000] -> 25[26000] via P2P/IPC comm 0x2b0c64000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 01/0 : 31[63000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2c04000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 00 : 32[4000] -> 33[26000] via P2P/IPC comm 0x2b2c04000ab0 nRanks 96
e07r2n17:31187:1570 [3] NCCL INFO Channel 00/0 : 79[63000] -> 80[4000] [send] via NET/IB/0 comm 0x2b137c000ab0 nRanks 96
e07r1n18:15189:19057 [3] NCCL INFO Channel 00/0 : 3[63000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6fbc000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 01/0 : 75[63000] -> 76[4000] [receive] via NET/IB/0 comm 0x2bb19c000ab0 nRanks 96
e07r2n16:26998:30253 [3] NCCL INFO Channel 00/0 : 75[63000] -> 76[4000] [send] via NET/IB/0 comm 0x2b5f04000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 01 : 84[4000] -> 85[26000] via P2P/IPC comm 0x2b5ad0000ab0 nRanks 96
e07r1n18:15185:19054 [0] NCCL INFO Channel 01/0 : 95[63000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b2ef8000ab0 nRanks 96
e07r1n18:15185:19054 [0] NCCL INFO Channel 00 : 0[4000] -> 1[26000] via P2P/IPC comm 0x2b2ef8000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 00 : 76[4000] -> 77[26000] via P2P/IPC comm 0x2bb19c000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 01/0 : 71[63000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e07r3n00:24490:27776 [0] NCCL INFO Channel 01 : 88[4000] -> 89[26000] via P2P/IPC comm 0x2b876c000ab0 nRanks 96
e07r3n01:6046:9750 [0] NCCL INFO Channel 01 : 92[4000] -> 93[26000] via P2P/IPC comm 0x2b0dd4000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 01/0 : 59[63000] -> 60[4000] [receive] via NET/IB/0 comm 0x2aec38000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 01 : 80[4000] -> 81[26000] via P2P/IPC comm 0x2b97a0000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 00 : 72[4000] -> 73[26000] via P2P/IPC comm 0x2b7ef4000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 00 : 60[4000] -> 61[26000] via P2P/IPC comm 0x2aec38000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 01 : 28[4000] -> 29[26000] via P2P/IPC comm 0x2ba624000ab0 nRanks 96
e07r2n03:21866:25086 [3] NCCL INFO Channel 00/0 : 23[63000] -> 24[4000] [send] via NET/IB/0 comm 0x2b6e40000ab0 nRanks 96
e07r2n14:22275:25517 [0] NCCL INFO Channel 01 : 64[4000] -> 65[26000] via P2P/IPC comm 0x2b6bd8000ab0 nRanks 96
e07r2n00:23103:26351 [3] NCCL INFO Channel 00/0 : 11[63000] -> 12[4000] [send] via NET/IB/0 comm 0x2b8604000ab0 nRanks 96
e07r1n19:30602:1034 [3] NCCL INFO Channel 00/0 : 7[63000] -> 8[4000] [send] via NET/IB/0 comm 0x2b2e08000ab0 nRanks 96
e07r2n01:629:4327 [3] NCCL INFO Channel 00/0 : 15[63000] -> 16[4000] [send] via NET/IB/0 comm 0x2ad0f4000ab0 nRanks 96
e07r2n15:28150:31327 [3] NCCL INFO Channel 00/0 : 71[63000] -> 72[4000] [send] via NET/IB/0 comm 0x2b3cb0000ab0 nRanks 96
e07r2n11:16529:19772 [3] NCCL INFO Channel 00/0 : 55[63000] -> 56[4000] [send] via NET/IB/0 comm 0x2b84d8000ab0 nRanks 96
e07r2n12:16039:18766 [3] NCCL INFO Channel 00/0 : 59[63000] -> 60[4000] [send] via NET/IB/0 comm 0x2b4134000ab0 nRanks 96
e07r1n18:15185:19054 [0] NCCL INFO Channel 01 : 0[4000] -> 1[26000] via P2P/IPC comm 0x2b2ef8000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 01/0 : 19[63000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3d8c000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 01/0 : 7[63000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7e4c000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 01/0 : 3[63000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b7088000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 01/0 : 11[63000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae9b8000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 01/0 : 51[63000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b86d4000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 00 : 52[4000] -> 53[26000] via P2P/IPC comm 0x2b86d4000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 01/0 : 55[63000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b858c000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 01 : 32[4000] -> 33[26000] via P2P/IPC comm 0x2b2c04000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 01 : 76[4000] -> 77[26000] via P2P/IPC comm 0x2bb19c000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 01 : 24[4000] -> 25[26000] via P2P/IPC comm 0x2b0c64000ab0 nRanks 96
e07r2n02:4760:8316 [3] NCCL INFO Channel 00/0 : 19[63000] -> 20[4000] [send] via NET/IB/0 comm 0x2b7558000ab0 nRanks 96
e07r2n09:23026:26303 [3] NCCL INFO Channel 00/0 : 47[63000] -> 48[4000] [send] via NET/IB/0 comm 0x2b4b4c000ab0 nRanks 96
e07r2n08:30380:1325 [3] NCCL INFO Channel 00/0 : 43[63000] -> 44[4000] [send] via NET/IB/0 comm 0x2b8818000ab0 nRanks 96
e07r2n07:11610:15050 [3] NCCL INFO Channel 00/0 : 39[63000] -> 40[4000] [send] via NET/IB/0 comm 0x2affe4000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 00 : 20[4000] -> 21[26000] via P2P/IPC comm 0x2b3d8c000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 00 : 8[4000] -> 9[26000] via P2P/IPC comm 0x2b7e4c000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 00 : 4[4000] -> 5[26000] via P2P/IPC comm 0x2b7088000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 00 : 12[4000] -> 13[26000] via P2P/IPC comm 0x2ae9b8000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 01 : 52[4000] -> 53[26000] via P2P/IPC comm 0x2b86d4000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 00 : 56[4000] -> 57[26000] via P2P/IPC comm 0x2b858c000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 01/0 : 15[63000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b71c4000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 00 : 16[4000] -> 17[26000] via P2P/IPC comm 0x2b71c4000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 01/0 : 43[63000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af9d0000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 01/0 : 39[63000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4274000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 01/0 : 35[63000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac9e0000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 01 : 60[4000] -> 61[26000] via P2P/IPC comm 0x2aec38000ab0 nRanks 96
e07r2n05:909:4280 [3] NCCL INFO Channel 00/0 : 31[63000] -> 32[4000] [send] via NET/IB/0 comm 0x2ac550000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 01 : 72[4000] -> 73[26000] via P2P/IPC comm 0x2b7ef4000ab0 nRanks 96
e07r3n00:24493:27774 [3] NCCL INFO Channel 00/0 : 91[63000] -> 92[4000] [send] via NET/IB/0 comm 0x2b0888000ab0 nRanks 96
e07r2n10:29373:32188 [3] NCCL INFO Channel 00/0 : 51[63000] -> 52[4000] [send] via NET/IB/0 comm 0x2b2da4000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 01 : 20[4000] -> 21[26000] via P2P/IPC comm 0x2b3d8c000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 01 : 8[4000] -> 9[26000] via P2P/IPC comm 0x2b7e4c000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 01 : 4[4000] -> 5[26000] via P2P/IPC comm 0x2b7088000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 01 : 12[4000] -> 13[26000] via P2P/IPC comm 0x2ae9b8000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 01 : 56[4000] -> 57[26000] via P2P/IPC comm 0x2b858c000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 01 : 16[4000] -> 17[26000] via P2P/IPC comm 0x2b71c4000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 00 : 44[4000] -> 45[26000] via P2P/IPC comm 0x2af9d0000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 00 : 40[4000] -> 41[26000] via P2P/IPC comm 0x2b4274000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 00 : 36[4000] -> 37[26000] via P2P/IPC comm 0x2ac9e0000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 01/0 : 47[63000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b2760000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 00 : 48[4000] -> 49[26000] via P2P/IPC comm 0x2b2760000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 01 : 44[4000] -> 45[26000] via P2P/IPC comm 0x2af9d0000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 01 : 40[4000] -> 41[26000] via P2P/IPC comm 0x2b4274000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 01 : 36[4000] -> 37[26000] via P2P/IPC comm 0x2ac9e0000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 01 : 48[4000] -> 49[26000] via P2P/IPC comm 0x2b2760000ab0 nRanks 96
e07r3n00:24491:27778 [1] NCCL INFO Connected all rings comm 0x2b9dcc000ab0 nRanks 96 busId 26000
e07r2n15:28147:31332 [1] NCCL INFO Connected all rings comm 0x2b1c40000ab0 nRanks 96 busId 26000
e07r2n17:31185:1575 [1] NCCL INFO Connected all rings comm 0x2ac95c000ab0 nRanks 96 busId 26000
e07r2n19:2109:5605 [1] NCCL INFO Connected all rings comm 0x2b3080000ab0 nRanks 96 busId 26000
e07r2n13:31207:1488 [1] NCCL INFO Connected all rings comm 0x2aedb4000ab0 nRanks 96 busId 26000
e07r3n01:6047:9752 [1] NCCL INFO Connected all rings comm 0x2ade98000ab0 nRanks 96 busId 26000
e07r2n18:25918:28644 [1] NCCL INFO Connected all rings comm 0x2b0bb8000ab0 nRanks 96 busId 26000
e07r2n00:23101:26354 [1] NCCL INFO Connected all rings comm 0x2ac104000ab0 nRanks 96 busId 26000
e07r2n14:22276:25518 [1] NCCL INFO Connected all rings comm 0x2af10c000ab0 nRanks 96 busId 26000
e07r2n05:907:4286 [1] NCCL INFO Connected all rings comm 0x2af92c000ab0 nRanks 96 busId 26000
e07r2n16:26996:30258 [1] NCCL INFO Connected all rings comm 0x2ae0fc000ab0 nRanks 96 busId 26000
e07r2n03:21864:25088 [1] NCCL INFO Connected all rings comm 0x2b34e0000ab0 nRanks 96 busId 26000
e07r2n04:1827:5193 [1] NCCL INFO Connected all rings comm 0x2ac224000ab0 nRanks 96 busId 26000
e07r3n00:24491:27778 [1] NCCL INFO Channel 00/0 : 84[4000] -> 89[26000] [receive] via NET/IB/0 comm 0x2b9dcc000ab0 nRanks 96
e07r2n19:2109:5605 [1] NCCL INFO Channel 01/0 : 85[26000] -> 88[4000] [send] via NET/IB/0 comm 0x2b3080000ab0 nRanks 96
e07r2n12:16037:18770 [1] NCCL INFO Connected all rings comm 0x2ae364000ab0 nRanks 96 busId 26000
e07r2n17:31185:1575 [1] NCCL INFO Channel 01/0 : 77[26000] -> 84[4000] [send] via NET/IB/0 comm 0x2ac95c000ab0 nRanks 96
e07r1n19:30600:1040 [1] NCCL INFO Connected all rings comm 0x2b48a0000ab0 nRanks 96 busId 26000
e07r2n01:627:4330 [1] NCCL INFO Connected all rings comm 0x2b7854000ab0 nRanks 96 busId 26000
e07r2n02:4758:8319 [1] NCCL INFO Connected all rings comm 0x2b8694000ab0 nRanks 96 busId 26000
e07r1n18:15187:19060 [1] NCCL INFO Connected all rings comm 0x2ac9ac000ab0 nRanks 96 busId 26000
e07r2n13:31207:1488 [1] NCCL INFO Channel 01/0 : 61[26000] -> 76[4000] [send] via NET/IB/0 comm 0x2aedb4000ab0 nRanks 96
e07r2n06:26609:29335 [1] NCCL INFO Connected all rings comm 0x2ae97c000ab0 nRanks 96 busId 26000
e07r2n05:907:4286 [1] NCCL INFO Channel 01/0 : 29[26000] -> 60[4000] [send] via NET/IB/0 comm 0x2af92c000ab0 nRanks 96
e07r2n07:11608:15052 [1] NCCL INFO Connected all rings comm 0x2aed18000ab0 nRanks 96 busId 26000
e07r2n04:1827:5193 [1] NCCL INFO Channel 00/0 : 20[4000] -> 25[26000] [receive] via NET/IB/0 comm 0x2ac224000ab0 nRanks 96
e07r2n11:16527:19776 [1] NCCL INFO Connected all rings comm 0x2aebfc000ab0 nRanks 96 busId 26000
e07r2n00:23101:26354 [1] NCCL INFO Channel 00/0 : 4[4000] -> 9[26000] [receive] via NET/IB/0 comm 0x2ac104000ab0 nRanks 96
e07r2n18:25918:28644 [1] NCCL INFO Channel 00/0 : 72[4000] -> 81[26000] [receive] via NET/IB/0 comm 0x2b0bb8000ab0 nRanks 96
e07r2n03:21864:25088 [1] NCCL INFO Channel 01/0 : 21[26000] -> 24[4000] [send] via NET/IB/0 comm 0x2b34e0000ab0 nRanks 96
e07r3n01:6047:9752 [1] NCCL INFO Channel 00 : 93[26000] -> 92[4000] via P2P/IPC comm 0x2ade98000ab0 nRanks 96
e07r2n14:22276:25518 [1] NCCL INFO Channel 00/0 : 32[4000] -> 65[26000] [receive] via NET/IB/0 comm 0x2af10c000ab0 nRanks 96
e07r2n16:26996:30258 [1] NCCL INFO Channel 00/0 : 68[4000] -> 73[26000] [receive] via NET/IB/0 comm 0x2ae0fc000ab0 nRanks 96
e07r1n19:30600:1040 [1] NCCL INFO Channel 01/0 : 5[26000] -> 8[4000] [send] via NET/IB/0 comm 0x2b48a0000ab0 nRanks 96
e07r2n01:627:4330 [1] NCCL INFO Channel 01/0 : 13[26000] -> 20[4000] [send] via NET/IB/0 comm 0x2b7854000ab0 nRanks 96
e07r2n12:16037:18770 [1] NCCL INFO Channel 00/0 : 52[4000] -> 57[26000] [receive] via NET/IB/0 comm 0x2ae364000ab0 nRanks 96
e07r3n01:6047:9752 [1] NCCL INFO Channel 01 : 93[26000] -> 92[4000] via P2P/IPC comm 0x2ade98000ab0 nRanks 96
e07r2n08:30378:1329 [1] NCCL INFO Connected all rings comm 0x2b2e18000ab0 nRanks 96 busId 26000
e07r2n09:23024:26305 [1] NCCL INFO Connected all rings comm 0x2b3c5c000ab0 nRanks 96 busId 26000
e07r2n02:4758:8319 [1] NCCL INFO Channel 00/0 : 8[4000] -> 17[26000] [receive] via NET/IB/0 comm 0x2b8694000ab0 nRanks 96
e07r2n07:11608:15052 [1] NCCL INFO Channel 01/0 : 37[26000] -> 40[4000] [send] via NET/IB/0 comm 0x2aed18000ab0 nRanks 96
e07r2n06:26609:29335 [1] NCCL INFO Channel 00/0 : 16[4000] -> 33[26000] [receive] via NET/IB/0 comm 0x2ae97c000ab0 nRanks 96
e07r1n18:15187:19060 [1] NCCL INFO Channel 00 : 1[26000] -> 0[4000] via P2P/IPC comm 0x2ac9ac000ab0 nRanks 96
e07r2n11:16527:19776 [1] NCCL INFO Channel 01/0 : 53[26000] -> 56[4000] [send] via NET/IB/0 comm 0x2aebfc000ab0 nRanks 96
e07r2n10:29371:32191 [1] NCCL INFO Connected all rings comm 0x2ad734000ab0 nRanks 96 busId 26000
e07r2n09:23024:26305 [1] NCCL INFO Channel 01/0 : 45[26000] -> 52[4000] [send] via NET/IB/0 comm 0x2b3c5c000ab0 nRanks 96
e07r1n18:15187:19060 [1] NCCL INFO Channel 01 : 1[26000] -> 0[4000] via P2P/IPC comm 0x2ac9ac000ab0 nRanks 96
e07r2n08:30378:1329 [1] NCCL INFO Channel 00/0 : 36[4000] -> 41[26000] [receive] via NET/IB/0 comm 0x2b2e18000ab0 nRanks 96
e07r2n10:29371:32191 [1] NCCL INFO Channel 00/0 : 40[4000] -> 49[26000] [receive] via NET/IB/0 comm 0x2ad734000ab0 nRanks 96
e07r2n13:31210:1485 [3] NCCL INFO Channel 01/0 : 63[63000] -> 64[4000] [send] via NET/IB/0 comm 0x2afab0000ab0 nRanks 96
e07r2n19:2111:5601 [3] NCCL INFO Channel 01/0 : 87[63000] -> 88[4000] [send] via NET/IB/0 comm 0x2b3334000ab0 nRanks 96
e07r3n01:6049:9748 [3] NCCL INFO Channel 01/0 : 95[63000] -> 0[4000] [send] via NET/IB/0 comm 0x2b508c000ab0 nRanks 96
e07r2n14:22278:25514 [3] NCCL INFO Channel 01/0 : 67[63000] -> 68[4000] [send] via NET/IB/0 comm 0x2b1220000ab0 nRanks 96
e07r2n18:25920:28641 [3] NCCL INFO Channel 01/0 : 83[63000] -> 84[4000] [send] via NET/IB/0 comm 0x2af828000ab0 nRanks 96
e07r2n04:1829:5188 [3] NCCL INFO Channel 01/0 : 27[63000] -> 28[4000] [send] via NET/IB/0 comm 0x2b68a8000ab0 nRanks 96
e07r2n06:26611:29331 [3] NCCL INFO Channel 01/0 : 35[63000] -> 36[4000] [send] via NET/IB/0 comm 0x2b62e4000ab0 nRanks 96
e07r2n17:31187:1570 [3] NCCL INFO Channel 01/0 : 79[63000] -> 80[4000] [send] via NET/IB/0 comm 0x2b137c000ab0 nRanks 96
e07r1n18:15189:19057 [3] NCCL INFO Channel 01/0 : 3[63000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6fbc000ab0 nRanks 96
e07r2n16:26998:30253 [3] NCCL INFO Channel 01/0 : 75[63000] -> 76[4000] [send] via NET/IB/0 comm 0x2b5f04000ab0 nRanks 96
e07r2n03:21866:25086 [3] NCCL INFO Channel 01/0 : 23[63000] -> 24[4000] [send] via NET/IB/0 comm 0x2b6e40000ab0 nRanks 96
e07r1n19:30602:1034 [3] NCCL INFO Channel 01/0 : 7[63000] -> 8[4000] [send] via NET/IB/0 comm 0x2b2e08000ab0 nRanks 96
e07r2n00:23103:26351 [3] NCCL INFO Channel 01/0 : 11[63000] -> 12[4000] [send] via NET/IB/0 comm 0x2b8604000ab0 nRanks 96
e07r2n01:629:4327 [3] NCCL INFO Channel 01/0 : 15[63000] -> 16[4000] [send] via NET/IB/0 comm 0x2ad0f4000ab0 nRanks 96
e07r2n12:16039:18766 [3] NCCL INFO Channel 01/0 : 59[63000] -> 60[4000] [send] via NET/IB/0 comm 0x2b4134000ab0 nRanks 96
e07r2n15:28150:31327 [3] NCCL INFO Channel 01/0 : 71[63000] -> 72[4000] [send] via NET/IB/0 comm 0x2b3cb0000ab0 nRanks 96
e07r2n11:16529:19772 [3] NCCL INFO Channel 01/0 : 55[63000] -> 56[4000] [send] via NET/IB/0 comm 0x2b84d8000ab0 nRanks 96
e07r2n02:4760:8316 [3] NCCL INFO Channel 01/0 : 19[63000] -> 20[4000] [send] via NET/IB/0 comm 0x2b7558000ab0 nRanks 96
e07r2n09:23026:26303 [3] NCCL INFO Channel 01/0 : 47[63000] -> 48[4000] [send] via NET/IB/0 comm 0x2b4b4c000ab0 nRanks 96
e07r2n07:11610:15050 [3] NCCL INFO Channel 01/0 : 39[63000] -> 40[4000] [send] via NET/IB/0 comm 0x2affe4000ab0 nRanks 96
e07r2n08:30380:1325 [3] NCCL INFO Channel 01/0 : 43[63000] -> 44[4000] [send] via NET/IB/0 comm 0x2b8818000ab0 nRanks 96
e07r2n05:909:4280 [3] NCCL INFO Channel 01/0 : 31[63000] -> 32[4000] [send] via NET/IB/0 comm 0x2ac550000ab0 nRanks 96
e07r3n00:24493:27774 [3] NCCL INFO Channel 01/0 : 91[63000] -> 92[4000] [send] via NET/IB/0 comm 0x2b0888000ab0 nRanks 96
e07r2n10:29373:32188 [3] NCCL INFO Channel 01/0 : 51[63000] -> 52[4000] [send] via NET/IB/0 comm 0x2b2da4000ab0 nRanks 96
e07r2n13:31209:1487 [2] NCCL INFO Connected all rings comm 0x2aba34000ab0 nRanks 96 busId 43000
e07r2n19:2110:5599 [2] NCCL INFO Connected all rings comm 0x2b9f58000ab0 nRanks 96 busId 43000
e07r2n14:22277:25513 [2] NCCL INFO Connected all rings comm 0x2b75dc000ab0 nRanks 96 busId 43000
e07r2n18:25919:28639 [2] NCCL INFO Connected all rings comm 0x2af1a8000ab0 nRanks 96 busId 43000
e07r2n06:26610:29330 [2] NCCL INFO Connected all rings comm 0x2b9594000ab0 nRanks 96 busId 43000
e07r2n17:31186:1568 [2] NCCL INFO Connected all rings comm 0x2b30ec000ab0 nRanks 96 busId 43000
e07r1n18:15188:19056 [2] NCCL INFO Connected all rings comm 0x2b6c7c000ab0 nRanks 96 busId 43000
e07r2n13:31209:1487 [2] NCCL INFO Channel 00 : 62[43000] -> 61[26000] via P2P/IPC comm 0x2aba34000ab0 nRanks 96
e07r3n01:6048:9746 [2] NCCL INFO Connected all rings comm 0x2b4198000ab0 nRanks 96 busId 43000
e07r2n19:2110:5599 [2] NCCL INFO Channel 00 : 86[43000] -> 85[26000] via P2P/IPC comm 0x2b9f58000ab0 nRanks 96
e07r2n14:22277:25513 [2] NCCL INFO Channel 00 : 66[43000] -> 65[26000] via P2P/IPC comm 0x2b75dc000ab0 nRanks 96
e07r2n00:23102:26350 [2] NCCL INFO Connected all rings comm 0x2ace64000ab0 nRanks 96 busId 43000
e07r2n15:28147:31332 [1] NCCL INFO Channel 01/0 : 69[26000] -> 72[4000] [send] via NET/IB/0 comm 0x2b1c40000ab0 nRanks 96
e07r2n03:21865:25084 [2] NCCL INFO Connected all rings comm 0x2b628c000ab0 nRanks 96 busId 43000
e07r2n04:1828:5186 [2] NCCL INFO Connected all rings comm 0x2b227c000ab0 nRanks 96 busId 43000
e07r2n18:25919:28639 [2] NCCL INFO Channel 00 : 82[43000] -> 81[26000] via P2P/IPC comm 0x2af1a8000ab0 nRanks 96
e07r2n19:2110:5599 [2] NCCL INFO Channel 01 : 86[43000] -> 85[26000] via P2P/IPC comm 0x2b9f58000ab0 nRanks 96
e07r2n13:31209:1487 [2] NCCL INFO Channel 01 : 62[43000] -> 61[26000] via P2P/IPC comm 0x2aba34000ab0 nRanks 96
e07r2n16:26997:30254 [2] NCCL INFO Connected all rings comm 0x2b23d4000ab0 nRanks 96 busId 43000
e07r2n06:26610:29330 [2] NCCL INFO Channel 00 : 34[43000] -> 33[26000] via P2P/IPC comm 0x2b9594000ab0 nRanks 96
e07r2n05:908:4281 [2] NCCL INFO Connected all rings comm 0x2b03d0000ab0 nRanks 96 busId 43000
e07r2n18:25919:28639 [2] NCCL INFO Channel 01 : 82[43000] -> 81[26000] via P2P/IPC comm 0x2af1a8000ab0 nRanks 96
e07r1n19:30601:1035 [2] NCCL INFO Connected all rings comm 0x2b9774000ab0 nRanks 96 busId 43000
e07r2n14:22277:25513 [2] NCCL INFO Channel 01 : 66[43000] -> 65[26000] via P2P/IPC comm 0x2b75dc000ab0 nRanks 96
e07r2n17:31186:1568 [2] NCCL INFO Channel 00 : 78[43000] -> 77[26000] via P2P/IPC comm 0x2b30ec000ab0 nRanks 96
e07r2n01:628:4325 [2] NCCL INFO Connected all rings comm 0x2aeb74000ab0 nRanks 96 busId 43000
e07r2n15:28148:31326 [2] NCCL INFO Connected all rings comm 0x2afcc0000ab0 nRanks 96 busId 43000
e07r3n01:6048:9746 [2] NCCL INFO Channel 00 : 94[43000] -> 93[26000] via P2P/IPC comm 0x2b4198000ab0 nRanks 96
e07r2n12:16038:18764 [2] NCCL INFO Connected all rings comm 0x2b9664000ab0 nRanks 96 busId 43000
e07r2n11:16528:19774 [2] NCCL INFO Connected all rings comm 0x2b8cf0000ab0 nRanks 96 busId 43000
e07r2n00:23102:26350 [2] NCCL INFO Channel 00 : 10[43000] -> 9[26000] via P2P/IPC comm 0x2ace64000ab0 nRanks 96
e07r2n06:26610:29330 [2] NCCL INFO Channel 01 : 34[43000] -> 33[26000] via P2P/IPC comm 0x2b9594000ab0 nRanks 96
e07r2n02:4759:8314 [2] NCCL INFO Connected all rings comm 0x2b3524000ab0 nRanks 96 busId 43000
e07r2n03:21865:25084 [2] NCCL INFO Channel 00 : 22[43000] -> 21[26000] via P2P/IPC comm 0x2b628c000ab0 nRanks 96
e07r1n18:15188:19056 [2] NCCL INFO Channel 00 : 2[43000] -> 1[26000] via P2P/IPC comm 0x2b6c7c000ab0 nRanks 96
e07r2n08:30379:1323 [2] NCCL INFO Connected all rings comm 0x2adfbc000ab0 nRanks 96 busId 43000
e07r2n04:1828:5186 [2] NCCL INFO Channel 00 : 26[43000] -> 25[26000] via P2P/IPC comm 0x2b227c000ab0 nRanks 96
e07r2n17:31186:1568 [2] NCCL INFO Channel 01 : 78[43000] -> 77[26000] via P2P/IPC comm 0x2b30ec000ab0 nRanks 96
e07r2n07:11609:15048 [2] NCCL INFO Connected all rings comm 0x2accec000ab0 nRanks 96 busId 43000
e07r3n00:24492:27773 [2] NCCL INFO Connected all rings comm 0x2b9934000ab0 nRanks 96 busId 43000
e07r3n01:6048:9746 [2] NCCL INFO Channel 01 : 94[43000] -> 93[26000] via P2P/IPC comm 0x2b4198000ab0 nRanks 96
e07r2n00:23102:26350 [2] NCCL INFO Channel 01 : 10[43000] -> 9[26000] via P2P/IPC comm 0x2ace64000ab0 nRanks 96
e07r2n09:23025:26302 [2] NCCL INFO Connected all rings comm 0x2b4748000ab0 nRanks 96 busId 43000
e07r2n03:21865:25084 [2] NCCL INFO Channel 01 : 22[43000] -> 21[26000] via P2P/IPC comm 0x2b628c000ab0 nRanks 96
e07r2n10:29372:32186 [2] NCCL INFO Connected all rings comm 0x2b4970000ab0 nRanks 96 busId 43000
e07r2n04:1828:5186 [2] NCCL INFO Channel 01 : 26[43000] -> 25[26000] via P2P/IPC comm 0x2b227c000ab0 nRanks 96
e07r1n18:15188:19056 [2] NCCL INFO Channel 01 : 2[43000] -> 1[26000] via P2P/IPC comm 0x2b6c7c000ab0 nRanks 96
e07r2n05:908:4281 [2] NCCL INFO Channel 00 : 30[43000] -> 29[26000] via P2P/IPC comm 0x2b03d0000ab0 nRanks 96
e07r2n16:26997:30254 [2] NCCL INFO Channel 00 : 74[43000] -> 73[26000] via P2P/IPC comm 0x2b23d4000ab0 nRanks 96
e07r1n19:30601:1035 [2] NCCL INFO Channel 00 : 6[43000] -> 5[26000] via P2P/IPC comm 0x2b9774000ab0 nRanks 96
e07r2n01:628:4325 [2] NCCL INFO Channel 00 : 14[43000] -> 13[26000] via P2P/IPC comm 0x2aeb74000ab0 nRanks 96
e07r2n15:28148:31326 [2] NCCL INFO Channel 00 : 70[43000] -> 69[26000] via P2P/IPC comm 0x2afcc0000ab0 nRanks 96
e07r2n05:908:4281 [2] NCCL INFO Channel 01 : 30[43000] -> 29[26000] via P2P/IPC comm 0x2b03d0000ab0 nRanks 96
e07r2n12:16038:18764 [2] NCCL INFO Channel 00 : 58[43000] -> 57[26000] via P2P/IPC comm 0x2b9664000ab0 nRanks 96
e07r2n08:30379:1323 [2] NCCL INFO Channel 00 : 42[43000] -> 41[26000] via P2P/IPC comm 0x2adfbc000ab0 nRanks 96
e07r2n16:26997:30254 [2] NCCL INFO Channel 01 : 74[43000] -> 73[26000] via P2P/IPC comm 0x2b23d4000ab0 nRanks 96
e07r2n11:16528:19774 [2] NCCL INFO Channel 00 : 54[43000] -> 53[26000] via P2P/IPC comm 0x2b8cf0000ab0 nRanks 96
e07r2n01:628:4325 [2] NCCL INFO Channel 01 : 14[43000] -> 13[26000] via P2P/IPC comm 0x2aeb74000ab0 nRanks 96
e07r1n19:30601:1035 [2] NCCL INFO Channel 01 : 6[43000] -> 5[26000] via P2P/IPC comm 0x2b9774000ab0 nRanks 96
e07r2n15:28148:31326 [2] NCCL INFO Channel 01 : 70[43000] -> 69[26000] via P2P/IPC comm 0x2afcc0000ab0 nRanks 96
e07r2n07:11609:15048 [2] NCCL INFO Channel 00 : 38[43000] -> 37[26000] via P2P/IPC comm 0x2accec000ab0 nRanks 96
e07r2n02:4759:8314 [2] NCCL INFO Channel 00 : 18[43000] -> 17[26000] via P2P/IPC comm 0x2b3524000ab0 nRanks 96
e07r3n00:24492:27773 [2] NCCL INFO Channel 00 : 90[43000] -> 89[26000] via P2P/IPC comm 0x2b9934000ab0 nRanks 96
e07r2n10:29372:32186 [2] NCCL INFO Channel 00 : 50[43000] -> 49[26000] via P2P/IPC comm 0x2b4970000ab0 nRanks 96
e07r2n11:16528:19774 [2] NCCL INFO Channel 01 : 54[43000] -> 53[26000] via P2P/IPC comm 0x2b8cf0000ab0 nRanks 96
e07r2n12:16038:18764 [2] NCCL INFO Channel 01 : 58[43000] -> 57[26000] via P2P/IPC comm 0x2b9664000ab0 nRanks 96
e07r2n09:23025:26302 [2] NCCL INFO Channel 00 : 46[43000] -> 45[26000] via P2P/IPC comm 0x2b4748000ab0 nRanks 96
e07r2n07:11609:15048 [2] NCCL INFO Channel 01 : 38[43000] -> 37[26000] via P2P/IPC comm 0x2accec000ab0 nRanks 96
e07r2n02:4759:8314 [2] NCCL INFO Channel 01 : 18[43000] -> 17[26000] via P2P/IPC comm 0x2b3524000ab0 nRanks 96
e07r2n08:30379:1323 [2] NCCL INFO Channel 01 : 42[43000] -> 41[26000] via P2P/IPC comm 0x2adfbc000ab0 nRanks 96
e07r2n10:29372:32186 [2] NCCL INFO Channel 01 : 50[43000] -> 49[26000] via P2P/IPC comm 0x2b4970000ab0 nRanks 96
e07r2n09:23025:26302 [2] NCCL INFO Channel 01 : 46[43000] -> 45[26000] via P2P/IPC comm 0x2b4748000ab0 nRanks 96
e07r3n00:24492:27773 [2] NCCL INFO Channel 01 : 90[43000] -> 89[26000] via P2P/IPC comm 0x2b9934000ab0 nRanks 96
e07r2n04:1829:5188 [3] NCCL INFO Connected all rings comm 0x2b68a8000ab0 nRanks 96 busId 63000
e07r2n04:1829:5188 [3] NCCL INFO Channel 00 : 27[63000] -> 26[43000] via P2P/IPC comm 0x2b68a8000ab0 nRanks 96
e07r2n04:1829:5188 [3] NCCL INFO Channel 01 : 27[63000] -> 26[43000] via P2P/IPC comm 0x2b68a8000ab0 nRanks 96
e07r2n18:25920:28641 [3] NCCL INFO Connected all rings comm 0x2af828000ab0 nRanks 96 busId 63000
e07r2n18:25920:28641 [3] NCCL INFO Channel 00 : 83[63000] -> 82[43000] via P2P/IPC comm 0x2af828000ab0 nRanks 96
e07r2n14:22278:25514 [3] NCCL INFO Connected all rings comm 0x2b1220000ab0 nRanks 96 busId 63000
e07r2n14:22278:25514 [3] NCCL INFO Channel 00 : 67[63000] -> 66[43000] via P2P/IPC comm 0x2b1220000ab0 nRanks 96
e07r2n18:25920:28641 [3] NCCL INFO Channel 01 : 83[63000] -> 82[43000] via P2P/IPC comm 0x2af828000ab0 nRanks 96
e07r2n13:31210:1485 [3] NCCL INFO Connected all rings comm 0x2afab0000ab0 nRanks 96 busId 63000
e07r2n13:31210:1485 [3] NCCL INFO Channel 00 : 63[63000] -> 62[43000] via P2P/IPC comm 0x2afab0000ab0 nRanks 96
e07r2n06:26611:29331 [3] NCCL INFO Connected all rings comm 0x2b62e4000ab0 nRanks 96 busId 63000
e07r2n06:26611:29331 [3] NCCL INFO Channel 00 : 35[63000] -> 34[43000] via P2P/IPC comm 0x2b62e4000ab0 nRanks 96
e07r2n19:2111:5601 [3] NCCL INFO Connected all rings comm 0x2b3334000ab0 nRanks 96 busId 63000
e07r2n14:22278:25514 [3] NCCL INFO Channel 01 : 67[63000] -> 66[43000] via P2P/IPC comm 0x2b1220000ab0 nRanks 96
e07r2n19:2111:5601 [3] NCCL INFO Channel 00 : 87[63000] -> 86[43000] via P2P/IPC comm 0x2b3334000ab0 nRanks 96
e07r2n12:16039:18766 [3] NCCL INFO Connected all rings comm 0x2b4134000ab0 nRanks 96 busId 63000
e07r2n12:16039:18766 [3] NCCL INFO Channel 00 : 59[63000] -> 58[43000] via P2P/IPC comm 0x2b4134000ab0 nRanks 96
e07r2n03:21866:25086 [3] NCCL INFO Connected all rings comm 0x2b6e40000ab0 nRanks 96 busId 63000
e07r2n01:629:4327 [3] NCCL INFO Connected all rings comm 0x2ad0f4000ab0 nRanks 96 busId 63000
e07r2n03:21866:25086 [3] NCCL INFO Channel 00 : 23[63000] -> 22[43000] via P2P/IPC comm 0x2b6e40000ab0 nRanks 96
e07r2n01:629:4327 [3] NCCL INFO Channel 00 : 15[63000] -> 14[43000] via P2P/IPC comm 0x2ad0f4000ab0 nRanks 96
e07r1n19:30602:1034 [3] NCCL INFO Connected all rings comm 0x2b2e08000ab0 nRanks 96 busId 63000
e07r1n19:30602:1034 [3] NCCL INFO Channel 00 : 7[63000] -> 6[43000] via P2P/IPC comm 0x2b2e08000ab0 nRanks 96
e07r2n06:26611:29331 [3] NCCL INFO Channel 01 : 35[63000] -> 34[43000] via P2P/IPC comm 0x2b62e4000ab0 nRanks 96
e07r2n13:31210:1485 [3] NCCL INFO Channel 01 : 63[63000] -> 62[43000] via P2P/IPC comm 0x2afab0000ab0 nRanks 96
e07r1n18:15189:19057 [3] NCCL INFO Connected all rings comm 0x2b6fbc000ab0 nRanks 96 busId 63000
e07r1n18:15189:19057 [3] NCCL INFO Channel 00 : 3[63000] -> 2[43000] via P2P/IPC comm 0x2b6fbc000ab0 nRanks 96
e07r2n19:2111:5601 [3] NCCL INFO Channel 01 : 87[63000] -> 86[43000] via P2P/IPC comm 0x2b3334000ab0 nRanks 96
e07r2n09:23026:26303 [3] NCCL INFO Connected all rings comm 0x2b4b4c000ab0 nRanks 96 busId 63000
e07r2n16:26998:30253 [3] NCCL INFO Connected all rings comm 0x2b5f04000ab0 nRanks 96 busId 63000
e07r2n01:629:4327 [3] NCCL INFO Channel 01 : 15[63000] -> 14[43000] via P2P/IPC comm 0x2ad0f4000ab0 nRanks 96
e07r2n09:23026:26303 [3] NCCL INFO Channel 00 : 47[63000] -> 46[43000] via P2P/IPC comm 0x2b4b4c000ab0 nRanks 96
e07r2n16:26998:30253 [3] NCCL INFO Channel 00 : 75[63000] -> 74[43000] via P2P/IPC comm 0x2b5f04000ab0 nRanks 96
e07r2n03:21866:25086 [3] NCCL INFO Channel 01 : 23[63000] -> 22[43000] via P2P/IPC comm 0x2b6e40000ab0 nRanks 96
e07r1n19:30602:1034 [3] NCCL INFO Channel 01 : 7[63000] -> 6[43000] via P2P/IPC comm 0x2b2e08000ab0 nRanks 96
e07r2n12:16039:18766 [3] NCCL INFO Channel 01 : 59[63000] -> 58[43000] via P2P/IPC comm 0x2b4134000ab0 nRanks 96
e07r1n18:15189:19057 [3] NCCL INFO Channel 01 : 3[63000] -> 2[43000] via P2P/IPC comm 0x2b6fbc000ab0 nRanks 96
e07r2n09:23026:26303 [3] NCCL INFO Channel 01 : 47[63000] -> 46[43000] via P2P/IPC comm 0x2b4b4c000ab0 nRanks 96
e07r2n07:11610:15050 [3] NCCL INFO Connected all rings comm 0x2affe4000ab0 nRanks 96 busId 63000
e07r2n02:4760:8316 [3] NCCL INFO Connected all rings comm 0x2b7558000ab0 nRanks 96 busId 63000
e07r2n16:26998:30253 [3] NCCL INFO Channel 01 : 75[63000] -> 74[43000] via P2P/IPC comm 0x2b5f04000ab0 nRanks 96
e07r2n07:11610:15050 [3] NCCL INFO Channel 00 : 39[63000] -> 38[43000] via P2P/IPC comm 0x2affe4000ab0 nRanks 96
e07r2n02:4760:8316 [3] NCCL INFO Channel 00 : 19[63000] -> 18[43000] via P2P/IPC comm 0x2b7558000ab0 nRanks 96
e07r2n11:16529:19772 [3] NCCL INFO Connected all rings comm 0x2b84d8000ab0 nRanks 96 busId 63000
e07r2n11:16529:19772 [3] NCCL INFO Channel 00 : 55[63000] -> 54[43000] via P2P/IPC comm 0x2b84d8000ab0 nRanks 96
e07r2n00:23103:26351 [3] NCCL INFO Connected all rings comm 0x2b8604000ab0 nRanks 96 busId 63000
e07r2n07:11610:15050 [3] NCCL INFO Channel 01 : 39[63000] -> 38[43000] via P2P/IPC comm 0x2affe4000ab0 nRanks 96
e07r2n00:23103:26351 [3] NCCL INFO Channel 00 : 11[63000] -> 10[43000] via P2P/IPC comm 0x2b8604000ab0 nRanks 96
e07r2n02:4760:8316 [3] NCCL INFO Channel 01 : 19[63000] -> 18[43000] via P2P/IPC comm 0x2b7558000ab0 nRanks 96
e07r2n04:1829:5188 [3] NCCL INFO Connected all trees comm 0x2b68a8000ab0 nRanks 96 busId 63000
e07r2n04:1829:5188 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28150:31327 [3] NCCL INFO Connected all rings comm 0x2b3cb0000ab0 nRanks 96 busId 63000
e07r2n04:1829:5188 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16529:19772 [3] NCCL INFO Channel 01 : 55[63000] -> 54[43000] via P2P/IPC comm 0x2b84d8000ab0 nRanks 96
e07r2n15:28150:31327 [3] NCCL INFO Channel 00 : 71[63000] -> 70[43000] via P2P/IPC comm 0x2b3cb0000ab0 nRanks 96
e07r2n17:31187:1570 [3] NCCL INFO Connected all rings comm 0x2b137c000ab0 nRanks 96 busId 63000
e07r2n17:31187:1570 [3] NCCL INFO Channel 00 : 79[63000] -> 78[43000] via P2P/IPC comm 0x2b137c000ab0 nRanks 96
e07r2n05:909:4280 [3] NCCL INFO Connected all rings comm 0x2ac550000ab0 nRanks 96 busId 63000
e07r2n10:29373:32188 [3] NCCL INFO Connected all rings comm 0x2b2da4000ab0 nRanks 96 busId 63000
e07r2n14:22275:25517 [0] NCCL INFO Connected all rings comm 0x2b6bd8000ab0 nRanks 96 busId 4000
e07r2n05:909:4280 [3] NCCL INFO Channel 00 : 31[63000] -> 30[43000] via P2P/IPC comm 0x2ac550000ab0 nRanks 96
e07r2n10:29373:32188 [3] NCCL INFO Channel 00 : 51[63000] -> 50[43000] via P2P/IPC comm 0x2b2da4000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Connected all rings comm 0x2b0c64000ab0 nRanks 96 busId 4000
e07r2n00:23103:26351 [3] NCCL INFO Channel 01 : 11[63000] -> 10[43000] via P2P/IPC comm 0x2b8604000ab0 nRanks 96
e07r2n15:28150:31327 [3] NCCL INFO Channel 01 : 71[63000] -> 70[43000] via P2P/IPC comm 0x2b3cb0000ab0 nRanks 96
e07r3n00:24493:27774 [3] NCCL INFO Connected all rings comm 0x2b0888000ab0 nRanks 96 busId 63000
e07r2n17:31187:1570 [3] NCCL INFO Channel 01 : 79[63000] -> 78[43000] via P2P/IPC comm 0x2b137c000ab0 nRanks 96
e07r3n00:24493:27774 [3] NCCL INFO Channel 00 : 91[63000] -> 90[43000] via P2P/IPC comm 0x2b0888000ab0 nRanks 96
e07r2n05:909:4280 [3] NCCL INFO Channel 01 : 31[63000] -> 30[43000] via P2P/IPC comm 0x2ac550000ab0 nRanks 96
e07r2n10:29373:32188 [3] NCCL INFO Channel 01 : 51[63000] -> 50[43000] via P2P/IPC comm 0x2b2da4000ab0 nRanks 96
e07r3n01:6049:9748 [3] NCCL INFO Connected all rings comm 0x2b508c000ab0 nRanks 96 busId 63000
e07r3n01:6049:9748 [3] NCCL INFO Channel 00 : 95[63000] -> 94[43000] via P2P/IPC comm 0x2b508c000ab0 nRanks 96
e07r2n18:25920:28641 [3] NCCL INFO Connected all trees comm 0x2af828000ab0 nRanks 96 busId 63000
e07r2n18:25920:28641 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n18:25920:28641 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23099:26356 [0] NCCL INFO Connected all rings comm 0x2b7e4c000ab0 nRanks 96 busId 4000
e07r2n14:22278:25514 [3] NCCL INFO Connected all trees comm 0x2b1220000ab0 nRanks 96 busId 63000
e07r3n00:24493:27774 [3] NCCL INFO Channel 01 : 91[63000] -> 90[43000] via P2P/IPC comm 0x2b0888000ab0 nRanks 96
e07r2n14:22278:25514 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6049:9748 [3] NCCL INFO Channel 01 : 95[63000] -> 94[43000] via P2P/IPC comm 0x2b508c000ab0 nRanks 96
e07r2n14:22278:25514 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n17:31184:1572 [0] NCCL INFO Connected all rings comm 0x2bb19c000ab0 nRanks 96 busId 4000
e07r2n06:26611:29331 [3] NCCL INFO Connected all trees comm 0x2b62e4000ab0 nRanks 96 busId 63000
e07r2n07:11607:15053 [0] NCCL INFO Connected all rings comm 0x2ac9e0000ab0 nRanks 96 busId 4000
e07r2n13:31210:1485 [3] NCCL INFO Connected all trees comm 0x2afab0000ab0 nRanks 96 busId 63000
e07r2n06:26611:29331 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31210:1485 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31210:1485 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n06:26611:29331 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28145:31330 [0] NCCL INFO Connected all rings comm 0x2b20a4000ab0 nRanks 96 busId 4000
e07r2n18:25917:28643 [0] NCCL INFO Connected all rings comm 0x2b97a0000ab0 nRanks 96 busId 4000
e07r2n19:2111:5601 [3] NCCL INFO Connected all trees comm 0x2b3334000ab0 nRanks 96 busId 63000
e07r1n19:30599:1037 [0] NCCL INFO Connected all rings comm 0x2b7088000ab0 nRanks 96 busId 4000
e07r2n01:629:4327 [3] NCCL INFO Connected all trees comm 0x2ad0f4000ab0 nRanks 96 busId 63000
e07r2n03:21866:25086 [3] NCCL INFO Connected all trees comm 0x2b6e40000ab0 nRanks 96 busId 63000
e07r2n05:906:4284 [0] NCCL INFO Connected all rings comm 0x2ba624000ab0 nRanks 96 busId 4000
e07r2n19:2111:5601 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n19:2111:5601 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n01:629:4327 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n01:629:4327 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n03:21866:25086 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n03:21866:25086 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n12:16039:18766 [3] NCCL INFO Connected all trees comm 0x2b4134000ab0 nRanks 96 busId 63000
e07r2n16:26995:30257 [0] NCCL INFO Connected all rings comm 0x2b7ef4000ab0 nRanks 96 busId 4000
e07r1n18:15189:19057 [3] NCCL INFO Connected all trees comm 0x2b6fbc000ab0 nRanks 96 busId 63000
e07r1n18:15189:19057 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n18:15189:19057 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n12:16039:18766 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n12:16039:18766 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26998:30253 [3] NCCL INFO Connected all trees comm 0x2b5f04000ab0 nRanks 96 busId 63000
e07r1n19:30602:1034 [3] NCCL INFO Connected all trees comm 0x2b2e08000ab0 nRanks 96 busId 63000
e07r3n00:24490:27776 [0] NCCL INFO Connected all rings comm 0x2b876c000ab0 nRanks 96 busId 4000
e07r2n16:26998:30253 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n16:26998:30253 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n19:30602:1034 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31206:1486 [0] NCCL INFO Connected all rings comm 0x2aec38000ab0 nRanks 96 busId 4000
e07r2n09:23026:26303 [3] NCCL INFO Connected all trees comm 0x2b4b4c000ab0 nRanks 96 busId 63000
e07r1n19:30602:1034 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n09:23026:26303 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n09:23026:26303 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n02:4756:8318 [0] NCCL INFO Connected all rings comm 0x2b71c4000ab0 nRanks 96 busId 4000
e07r2n12:16035:18767 [0] NCCL INFO Connected all rings comm 0x2b858c000ab0 nRanks 96 busId 4000
e07r3n01:6046:9750 [0] NCCL INFO Connected all rings comm 0x2b0dd4000ab0 nRanks 96 busId 4000
e07r2n02:4760:8316 [3] NCCL INFO Connected all trees comm 0x2b7558000ab0 nRanks 96 busId 63000
e07r2n02:4760:8316 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4760:8316 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n07:11610:15050 [3] NCCL INFO Connected all trees comm 0x2affe4000ab0 nRanks 96 busId 63000
e07r2n07:11610:15050 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11610:15050 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n03:21863:25090 [0] NCCL INFO Connected all rings comm 0x2b3d8c000ab0 nRanks 96 busId 4000
e07r1n18:15185:19054 [0] NCCL INFO Connected all rings comm 0x2b2ef8000ab0 nRanks 96 busId 4000
e07r1n18:15188:19056 [2] NCCL INFO Connected all trees comm 0x2b6c7c000ab0 nRanks 96 busId 43000
e07r1n18:15188:19056 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n18:15188:19056 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16529:19772 [3] NCCL INFO Connected all trees comm 0x2b84d8000ab0 nRanks 96 busId 63000
e07r2n10:29369:32190 [0] NCCL INFO Connected all rings comm 0x2b2760000ab0 nRanks 96 busId 4000
e07r2n11:16529:19772 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16529:19772 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28150:31327 [3] NCCL INFO Connected all trees comm 0x2b3cb0000ab0 nRanks 96 busId 63000
e07r2n15:28150:31327 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28150:31327 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23103:26351 [3] NCCL INFO Connected all trees comm 0x2b8604000ab0 nRanks 96 busId 63000
e07r2n00:23103:26351 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23103:26351 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n17:31187:1570 [3] NCCL INFO Connected all trees comm 0x2b137c000ab0 nRanks 96 busId 63000
e07r2n17:31187:1570 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31187:1570 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16526:19775 [0] NCCL INFO Connected all rings comm 0x2b86d4000ab0 nRanks 96 busId 4000
e07r2n10:29373:32188 [3] NCCL INFO Connected all trees comm 0x2b2da4000ab0 nRanks 96 busId 63000
e07r2n10:29373:32188 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:909:4280 [3] NCCL INFO Connected all trees comm 0x2ac550000ab0 nRanks 96 busId 63000
e07r2n10:29373:32188 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n05:909:4280 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:909:4280 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24493:27774 [3] NCCL INFO Connected all trees comm 0x2b0888000ab0 nRanks 96 busId 63000
e07r3n00:24493:27774 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n00:24493:27774 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n01:6049:9748 [3] NCCL INFO Connected all trees comm 0x2b508c000ab0 nRanks 96 busId 63000
e07r3n01:6049:9748 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6049:9748 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n01:6048:9746 [2] NCCL INFO Connected all trees comm 0x2b4198000ab0 nRanks 96 busId 43000
e07r3n01:6048:9746 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6048:9746 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n06:26608:29334 [0] NCCL INFO Connected all rings comm 0x2b2c04000ab0 nRanks 96 busId 4000
e07r2n01:626:4329 [0] NCCL INFO Connected all rings comm 0x2ae9b8000ab0 nRanks 96 busId 4000
e07r2n19:2107:5602 [0] NCCL INFO Connected all rings comm 0x2b5ad0000ab0 nRanks 96 busId 4000
e07r2n08:30377:1327 [0] NCCL INFO Connected all rings comm 0x2b4274000ab0 nRanks 96 busId 4000
e07r2n08:30380:1325 [3] NCCL INFO Connected all rings comm 0x2b8818000ab0 nRanks 96 busId 63000
e07r2n08:30380:1325 [3] NCCL INFO Channel 00 : 43[63000] -> 42[43000] via P2P/IPC comm 0x2b8818000ab0 nRanks 96
e07r2n08:30380:1325 [3] NCCL INFO Channel 01 : 43[63000] -> 42[43000] via P2P/IPC comm 0x2b8818000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Connected all rings comm 0x2af9d0000ab0 nRanks 96 busId 4000
e07r2n08:30380:1325 [3] NCCL INFO Connected all trees comm 0x2b8818000ab0 nRanks 96 busId 63000
e07r2n08:30380:1325 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n08:30380:1325 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22275:25517 [0] NCCL INFO Channel 01/0 : 64[4000] -> 68[4000] [send] via NET/IB/0 comm 0x2b6bd8000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 01/0 : 21[26000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b0c64000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 01/0 : 5[26000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7e4c000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 01/0 : 32[4000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac9e0000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 00/0 : 24[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba624000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 01/0 : 80[4000] -> 84[4000] [send] via NET/IB/0 comm 0x2b97a0000ab0 nRanks 96
e07r3n00:24490:27776 [0] NCCL INFO Channel 01/0 : 85[26000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b876c000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 00/0 : 72[4000] -> 76[4000] [receive] via NET/IB/0 comm 0x2bb19c000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 00/0 : 56[4000] -> 60[4000] [receive] via NET/IB/0 comm 0x2aec38000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 01/0 : 69[26000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 01/0 : 64[4000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b20a4000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 01/0 : 0[4000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b7088000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 01/0 : 16[4000] -> 20[4000] [send] via NET/IB/0 comm 0x2b71c4000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 01/0 : 53[26000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b858c000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 01/0 : 48[4000] -> 52[4000] [send] via NET/IB/0 comm 0x2b2760000ab0 nRanks 96
e07r3n01:6046:9750 [0] NCCL INFO Channel 00/0 : 88[4000] -> 92[4000] [receive] via NET/IB/0 comm 0x2b0dd4000ab0 nRanks 96
e07r1n18:15185:19054 [0] NCCL INFO Channel 01/0 : 0[4000] -> 4[4000] [send] via NET/IB/0 comm 0x2b2ef8000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 01/0 : 16[4000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3d8c000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 01/0 : 48[4000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b86d4000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 01/0 : 32[4000] -> 36[4000] [send] via NET/IB/0 comm 0x2b2c04000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 00/0 : 8[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae9b8000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 01/0 : 80[4000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5ad0000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 01/0 : 37[26000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4274000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 00/0 : 40[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af9d0000ab0 nRanks 96
e07r2n03:21864:25088 [1] NCCL INFO Channel 01/0 : 24[4000] -> 21[26000] [receive] via NET/IB/0 comm 0x2b34e0000ab0 nRanks 96
e07r2n19:2109:5605 [1] NCCL INFO Channel 01/0 : 88[4000] -> 85[26000] [receive] via NET/IB/0 comm 0x2b3080000ab0 nRanks 96
e07r2n14:22275:25517 [0] NCCL INFO Channel 00/0 : 64[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b6bd8000ab0 nRanks 96
e07r2n15:28147:31332 [1] NCCL INFO Channel 01/0 : 72[4000] -> 69[26000] [receive] via NET/IB/0 comm 0x2b1c40000ab0 nRanks 96
e07r1n19:30600:1040 [1] NCCL INFO Channel 01/0 : 8[4000] -> 5[26000] [receive] via NET/IB/0 comm 0x2b48a0000ab0 nRanks 96
e07r2n11:16527:19776 [1] NCCL INFO Channel 01/0 : 56[4000] -> 53[26000] [receive] via NET/IB/0 comm 0x2aebfc000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 00/0 : 48[4000] -> 56[4000] [send] via NET/IB/0 comm 0x2b2760000ab0 nRanks 96
e07r1n18:15185:19054 [0] NCCL INFO Channel 00/0 : 64[4000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b2ef8000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 00/0 : 16[4000] -> 24[4000] [send] via NET/IB/0 comm 0x2b71c4000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 00/0 : 32[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2b2c04000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 00/0 : 68[4000] -> 73[26000] [send] via NET/IB/0 comm 0x2b20a4000ab0 nRanks 96
e07r3n00:24490:27776 [0] NCCL INFO Channel 00/0 : 88[4000] -> 92[4000] [send] via NET/IB/0 comm 0x2b876c000ab0 nRanks 96
e07r2n07:11608:15052 [1] NCCL INFO Channel 01/0 : 40[4000] -> 37[26000] [receive] via NET/IB/0 comm 0x2aed18000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 00/0 : 80[4000] -> 88[4000] [send] via NET/IB/0 comm 0x2b97a0000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 00/0 : 24[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2b0c64000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 00/0 : 20[4000] -> 25[26000] [send] via NET/IB/0 comm 0x2b3d8c000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 00/0 : 4[4000] -> 9[26000] [send] via NET/IB/0 comm 0x2b7088000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 00/0 : 36[4000] -> 41[26000] [send] via NET/IB/0 comm 0x2ac9e0000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 00/0 : 8[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2b7e4c000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 00/0 : 84[4000] -> 89[26000] [send] via NET/IB/0 comm 0x2b5ad0000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 00/0 : 52[4000] -> 57[26000] [send] via NET/IB/0 comm 0x2b86d4000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 00/0 : 40[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2b4274000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 00/0 : 72[4000] -> 76[4000] [send] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 00/0 : 56[4000] -> 60[4000] [send] via NET/IB/0 comm 0x2b858c000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 01/0 : 68[4000] -> 76[4000] [send] via NET/IB/0 comm 0x2b20a4000ab0 nRanks 96
e07r3n00:24490:27776 [0] NCCL INFO Channel 00/0 : 80[4000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b876c000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 00/0 : 16[4000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b0c64000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 01/0 : 13[26000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3d8c000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 01/0 : 4[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2b7088000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 01/0 : 36[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2ac9e0000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 00/0 : 8[4000] -> 17[26000] [send] via NET/IB/0 comm 0x2b7e4c000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 01/0 : 77[26000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5ad0000ab0 nRanks 96
e07r2n04:1827:5193 [1] NCCL INFO Channel 00/0 : 25[26000] -> 20[4000] [send] via NET/IB/0 comm 0x2ac224000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 00/0 : 72[4000] -> 81[26000] [send] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 01/0 : 45[26000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b86d4000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 00/0 : 40[4000] -> 49[26000] [send] via NET/IB/0 comm 0x2b4274000ab0 nRanks 96
e07r2n16:26996:30258 [1] NCCL INFO Channel 00/0 : 73[26000] -> 68[4000] [send] via NET/IB/0 comm 0x2ae0fc000ab0 nRanks 96
e07r2n08:30378:1329 [1] NCCL INFO Channel 00/0 : 41[26000] -> 36[4000] [send] via NET/IB/0 comm 0x2b2e18000ab0 nRanks 96
e07r3n01:6046:9750 [0] NCCL INFO Channel 01/0 : 92[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2b0dd4000ab0 nRanks 96
e07r3n00:24491:27778 [1] NCCL INFO Channel 00/0 : 89[26000] -> 84[4000] [send] via NET/IB/0 comm 0x2b9dcc000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 01/0 : 12[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba624000ab0 nRanks 96
e07r2n00:23101:26354 [1] NCCL INFO Channel 00/0 : 9[26000] -> 4[4000] [send] via NET/IB/0 comm 0x2ac104000ab0 nRanks 96
e07r2n12:16037:18770 [1] NCCL INFO Channel 00/0 : 57[26000] -> 52[4000] [send] via NET/IB/0 comm 0x2ae364000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 01/0 : 4[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae9b8000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 00/0 : 48[4000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b858c000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 01/0 : 36[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af9d0000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 01/0 : 68[4000] -> 76[4000] [receive] via NET/IB/0 comm 0x2bb19c000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 01/0 : 44[4000] -> 60[4000] [receive] via NET/IB/0 comm 0x2aec38000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 00/0 : 16[4000] -> 33[26000] [send] via NET/IB/0 comm 0x2b71c4000ab0 nRanks 96
e07r2n01:627:4330 [1] NCCL INFO Channel 01/0 : 20[4000] -> 13[26000] [receive] via NET/IB/0 comm 0x2b7854000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 00/0 : 17[26000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7e4c000ab0 nRanks 96
e07r2n17:31185:1575 [1] NCCL INFO Channel 01/0 : 84[4000] -> 77[26000] [receive] via NET/IB/0 comm 0x2ac95c000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 00/0 : 81[26000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 00/0 : 49[26000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4274000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 01/0 : 20[4000] -> 13[26000] [send] via NET/IB/0 comm 0x2b3d8c000ab0 nRanks 96
e07r2n02:4758:8319 [1] NCCL INFO Channel 00/0 : 17[26000] -> 8[4000] [send] via NET/IB/0 comm 0x2b8694000ab0 nRanks 96
e07r3n00:24490:27776 [0] NCCL INFO Channel 00/0 : 88[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b876c000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 00/0 : 24[4000] -> 16[4000] [send] via NET/IB/0 comm 0x2b0c64000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 00/0 : 64[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b97a0000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 01/0 : 84[4000] -> 77[26000] [send] via NET/IB/0 comm 0x2b5ad0000ab0 nRanks 96
e07r2n18:25918:28644 [1] NCCL INFO Channel 00/0 : 81[26000] -> 72[4000] [send] via NET/IB/0 comm 0x2b0bb8000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 01/0 : 12[4000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b7088000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 00/0 : 32[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b2760000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 01/0 : 52[4000] -> 45[26000] [send] via NET/IB/0 comm 0x2b86d4000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 01/0 : 44[4000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac9e0000ab0 nRanks 96
e07r2n10:29371:32191 [1] NCCL INFO Channel 00/0 : 49[26000] -> 40[4000] [send] via NET/IB/0 comm 0x2ad734000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 01/0 : 76[4000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b20a4000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 01/0 : 12[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2ae9b8000ab0 nRanks 96
e07r2n09:23024:26305 [1] NCCL INFO Channel 01/0 : 52[4000] -> 45[26000] [receive] via NET/IB/0 comm 0x2b3c5c000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 01/0 : 44[4000] -> 60[4000] [send] via NET/IB/0 comm 0x2af9d0000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 01/0 : 61[26000] -> 76[4000] [receive] via NET/IB/0 comm 0x2bb19c000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 00/0 : 56[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2b858c000ab0 nRanks 96
e07r2n02:4758:8319 [1] NCCL INFO Channel 00 : 17[26000] -> 16[4000] via P2P/IPC comm 0x2b8694000ab0 nRanks 96
e07r2n02:4758:8319 [1] NCCL INFO Channel 01 : 17[26000] -> 16[4000] via P2P/IPC comm 0x2b8694000ab0 nRanks 96
e07r2n18:25918:28644 [1] NCCL INFO Channel 00 : 81[26000] -> 80[4000] via P2P/IPC comm 0x2b0bb8000ab0 nRanks 96
e07r2n18:25918:28644 [1] NCCL INFO Channel 01 : 81[26000] -> 80[4000] via P2P/IPC comm 0x2b0bb8000ab0 nRanks 96
e07r2n02:4759:8314 [2] NCCL INFO Connected all trees comm 0x2b3524000ab0 nRanks 96 busId 43000
e07r2n02:4759:8314 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4759:8314 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n10:29371:32191 [1] NCCL INFO Channel 00 : 49[26000] -> 48[4000] via P2P/IPC comm 0x2ad734000ab0 nRanks 96
e07r2n10:29371:32191 [1] NCCL INFO Channel 01 : 49[26000] -> 48[4000] via P2P/IPC comm 0x2ad734000ab0 nRanks 96
e07r2n01:627:4330 [1] NCCL INFO Channel 00 : 13[26000] -> 12[4000] via P2P/IPC comm 0x2b7854000ab0 nRanks 96
e07r2n18:25919:28639 [2] NCCL INFO Connected all trees comm 0x2af1a8000ab0 nRanks 96 busId 43000
e07r2n18:25919:28639 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n18:25919:28639 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n01:627:4330 [1] NCCL INFO Channel 01 : 13[26000] -> 12[4000] via P2P/IPC comm 0x2b7854000ab0 nRanks 96
e07r2n17:31185:1575 [1] NCCL INFO Channel 00 : 77[26000] -> 76[4000] via P2P/IPC comm 0x2ac95c000ab0 nRanks 96
e07r2n17:31185:1575 [1] NCCL INFO Channel 01 : 77[26000] -> 76[4000] via P2P/IPC comm 0x2ac95c000ab0 nRanks 96
e07r2n10:29372:32186 [2] NCCL INFO Connected all trees comm 0x2b4970000ab0 nRanks 96 busId 43000
e07r2n10:29372:32186 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n10:29372:32186 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n01:628:4325 [2] NCCL INFO Connected all trees comm 0x2aeb74000ab0 nRanks 96 busId 43000
e07r2n01:628:4325 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n01:628:4325 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n17:31186:1568 [2] NCCL INFO Connected all trees comm 0x2b30ec000ab0 nRanks 96 busId 43000
e07r2n17:31186:1568 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31186:1568 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n09:23024:26305 [1] NCCL INFO Channel 00 : 45[26000] -> 44[4000] via P2P/IPC comm 0x2b3c5c000ab0 nRanks 96
e07r2n09:23024:26305 [1] NCCL INFO Channel 01 : 45[26000] -> 44[4000] via P2P/IPC comm 0x2b3c5c000ab0 nRanks 96
e07r2n09:23025:26302 [2] NCCL INFO Connected all trees comm 0x2b4748000ab0 nRanks 96 busId 43000
e07r2n09:23025:26302 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n09:23025:26302 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n02:4756:8318 [0] NCCL INFO Channel 00/0 : 33[26000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b71c4000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 00/0 : 25[26000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b3d8c000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 00/0 : 89[26000] -> 84[4000] [receive] via NET/IB/0 comm 0x2b5ad0000ab0 nRanks 96
e07r2n14:22275:25517 [0] NCCL INFO Channel 00/0 : 64[4000] -> 0[4000] [send] via NET/IB/0 comm 0x2b6bd8000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 00/0 : 32[4000] -> 65[26000] [send] via NET/IB/0 comm 0x2b2c04000ab0 nRanks 96
e07r2n06:26609:29335 [1] NCCL INFO Channel 00/0 : 33[26000] -> 16[4000] [send] via NET/IB/0 comm 0x2ae97c000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 00/0 : 12[4000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b7e4c000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 01/0 : 28[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2ae9b8000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 00/0 : 80[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b97a0000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 00/0 : 48[4000] -> 32[4000] [send] via NET/IB/0 comm 0x2b2760000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 00/0 : 76[4000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 00/0 : 57[26000] -> 52[4000] [receive] via NET/IB/0 comm 0x2b86d4000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 01/0 : 60[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2af9d0000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 00/0 : 44[4000] -> 40[4000] [receive] via NET/IB/0 comm 0x2b4274000ab0 nRanks 96
e07r2n13:31207:1488 [1] NCCL INFO Channel 01/0 : 76[4000] -> 61[26000] [receive] via NET/IB/0 comm 0x2aedb4000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 01/0 : 92[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2ba624000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 01/0 : 29[26000] -> 60[4000] [receive] via NET/IB/0 comm 0x2aec38000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 01/0 : 76[4000] -> 61[26000] [send] via NET/IB/0 comm 0x2bb19c000ab0 nRanks 96
e07r2n04:1827:5193 [1] NCCL INFO Channel 00 : 25[26000] -> 24[4000] via P2P/IPC comm 0x2ac224000ab0 nRanks 96
e07r2n04:1827:5193 [1] NCCL INFO Channel 01 : 25[26000] -> 24[4000] via P2P/IPC comm 0x2ac224000ab0 nRanks 96
e07r3n00:24491:27778 [1] NCCL INFO Channel 00 : 89[26000] -> 88[4000] via P2P/IPC comm 0x2b9dcc000ab0 nRanks 96
e07r2n06:26609:29335 [1] NCCL INFO Channel 00 : 33[26000] -> 32[4000] via P2P/IPC comm 0x2ae97c000ab0 nRanks 96
e07r3n00:24491:27778 [1] NCCL INFO Channel 01 : 89[26000] -> 88[4000] via P2P/IPC comm 0x2b9dcc000ab0 nRanks 96
e07r2n04:1828:5186 [2] NCCL INFO Connected all trees comm 0x2b227c000ab0 nRanks 96 busId 43000
e07r2n04:1828:5186 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n06:26609:29335 [1] NCCL INFO Channel 01 : 33[26000] -> 32[4000] via P2P/IPC comm 0x2ae97c000ab0 nRanks 96
e07r2n04:1828:5186 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24492:27773 [2] NCCL INFO Connected all trees comm 0x2b9934000ab0 nRanks 96 busId 43000
e07r3n00:24492:27773 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n00:24492:27773 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n06:26610:29330 [2] NCCL INFO Connected all trees comm 0x2b9594000ab0 nRanks 96 busId 43000
e07r2n06:26610:29330 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n06:26610:29330 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n12:16037:18770 [1] NCCL INFO Channel 00 : 57[26000] -> 56[4000] via P2P/IPC comm 0x2ae364000ab0 nRanks 96
e07r2n12:16037:18770 [1] NCCL INFO Channel 01 : 57[26000] -> 56[4000] via P2P/IPC comm 0x2ae364000ab0 nRanks 96
e07r2n12:16038:18764 [2] NCCL INFO Connected all trees comm 0x2b9664000ab0 nRanks 96 busId 43000
e07r2n12:16038:18764 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n12:16038:18764 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n13:31207:1488 [1] NCCL INFO Channel 00 : 61[26000] -> 60[4000] via P2P/IPC comm 0x2aedb4000ab0 nRanks 96
e07r2n13:31207:1488 [1] NCCL INFO Channel 01 : 61[26000] -> 60[4000] via P2P/IPC comm 0x2aedb4000ab0 nRanks 96
e07r2n13:31209:1487 [2] NCCL INFO Connected all trees comm 0x2aba34000ab0 nRanks 96 busId 43000
e07r2n13:31209:1487 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31209:1487 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22275:25517 [0] NCCL INFO Channel 00/0 : 0[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b6bd8000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 00/0 : 65[26000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2c04000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO Channel 01/0 : 20[4000] -> 16[4000] [send] via NET/IB/0 comm 0x2b3d8c000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Channel 01/0 : 84[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b5ad0000ab0 nRanks 96
e07r1n18:15185:19054 [0] NCCL INFO Channel 00/0 : 0[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b2ef8000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 00/0 : 24[4000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b71c4000ab0 nRanks 96
e07r2n14:22276:25518 [1] NCCL INFO Channel 00/0 : 65[26000] -> 32[4000] [send] via NET/IB/0 comm 0x2af10c000ab0 nRanks 96
e07r2n05:907:4286 [1] NCCL INFO Channel 01/0 : 60[4000] -> 29[26000] [receive] via NET/IB/0 comm 0x2af92c000ab0 nRanks 96
e07r3n01:6046:9750 [0] NCCL INFO Channel 01/0 : 28[4000] -> 92[4000] [receive] via NET/IB/0 comm 0x2b0dd4000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 01/0 : 76[4000] -> 68[4000] [send] via NET/IB/0 comm 0x2bb19c000ab0 nRanks 96
e07r2n11:16526:19775 [0] NCCL INFO Channel 01/0 : 52[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2b86d4000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 01/0 : 28[4000] -> 92[4000] [send] via NET/IB/0 comm 0x2ba624000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 01/0 : 60[4000] -> 29[26000] [send] via NET/IB/0 comm 0x2aec38000ab0 nRanks 96
e07r2n14:22276:25518 [1] NCCL INFO Channel 00 : 65[26000] -> 64[4000] via P2P/IPC comm 0x2af10c000ab0 nRanks 96
e07r2n14:22276:25518 [1] NCCL INFO Channel 01 : 65[26000] -> 64[4000] via P2P/IPC comm 0x2af10c000ab0 nRanks 96
e07r2n14:22277:25513 [2] NCCL INFO Connected all trees comm 0x2b75dc000ab0 nRanks 96 busId 43000
e07r2n14:22277:25513 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n14:22277:25513 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n05:907:4286 [1] NCCL INFO Channel 00 : 29[26000] -> 28[4000] via P2P/IPC comm 0x2af92c000ab0 nRanks 96
e07r2n05:907:4286 [1] NCCL INFO Channel 01 : 29[26000] -> 28[4000] via P2P/IPC comm 0x2af92c000ab0 nRanks 96
e07r2n05:908:4281 [2] NCCL INFO Connected all trees comm 0x2b03d0000ab0 nRanks 96 busId 43000
e07r2n05:908:4281 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:908:4281 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n18:15185:19054 [0] NCCL INFO Channel 01/0 : 4[4000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b2ef8000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 00/0 : 28[4000] -> 24[4000] [receive] via NET/IB/0 comm 0x2b0c64000ab0 nRanks 96
e07r2n14:22275:25517 [0] NCCL INFO Channel 00/0 : 80[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b6bd8000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Channel 00/0 : 76[4000] -> 72[4000] [send] via NET/IB/0 comm 0x2bb19c000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Channel 01/0 : 20[4000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b71c4000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 00/0 : 48[4000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2c04000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 01/0 : 28[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2ba624000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 01/0 : 60[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2aec38000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 00/0 : 73[26000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b20a4000ab0 nRanks 96
e07r3n01:6046:9750 [0] NCCL INFO Channel 00/0 : 92[4000] -> 88[4000] [send] via NET/IB/0 comm 0x2b0dd4000ab0 nRanks 96
e07r2n17:31184:1572 [0] NCCL INFO Connected all trees comm 0x2bb19c000ab0 nRanks 96 busId 4000
e07r2n17:31184:1572 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31184:1572 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n17:31185:1575 [1] NCCL INFO Connected all trees comm 0x2ac95c000ab0 nRanks 96 busId 26000
e07r2n17:31185:1575 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n17:31185:1575 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26996:30258 [1] NCCL INFO Channel 00 : 73[26000] -> 72[4000] via P2P/IPC comm 0x2ae0fc000ab0 nRanks 96
e07r2n16:26996:30258 [1] NCCL INFO Channel 01 : 73[26000] -> 72[4000] via P2P/IPC comm 0x2ae0fc000ab0 nRanks 96
e07r2n02:4756:8318 [0] NCCL INFO Connected all trees comm 0x2b71c4000ab0 nRanks 96 busId 4000
e07r2n02:4756:8318 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4756:8318 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n02:4758:8319 [1] NCCL INFO Connected all trees comm 0x2b8694000ab0 nRanks 96 busId 26000
e07r2n02:4758:8319 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n02:4758:8319 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26997:30254 [2] NCCL INFO Connected all trees comm 0x2b23d4000ab0 nRanks 96 busId 43000
e07r2n16:26997:30254 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n16:26997:30254 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n18:25917:28643 [0] NCCL INFO Channel 00/0 : 88[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b97a0000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Channel 00/0 : 28[4000] -> 24[4000] [send] via NET/IB/0 comm 0x2ba624000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 00/0 : 56[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b2760000ab0 nRanks 96
e07r2n13:31206:1486 [0] NCCL INFO Channel 00/0 : 60[4000] -> 56[4000] [send] via NET/IB/0 comm 0x2aec38000ab0 nRanks 96
e07r2n14:22275:25517 [0] NCCL INFO Channel 01/0 : 68[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b6bd8000ab0 nRanks 96
e07r2n16:26995:30257 [0] NCCL INFO Channel 01/0 : 72[4000] -> 69[26000] [send] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e07r2n06:26608:29334 [0] NCCL INFO Channel 01/0 : 36[4000] -> 32[4000] [receive] via NET/IB/0 comm 0x2b2c04000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 01/0 : 12[4000] -> 4[4000] [send] via NET/IB/0 comm 0x2ae9b8000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Channel 01/0 : 68[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b20a4000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 01/0 : 44[4000] -> 36[4000] [send] via NET/IB/0 comm 0x2af9d0000ab0 nRanks 96
e07r2n05:906:4284 [0] NCCL INFO Connected all trees comm 0x2ba624000ab0 nRanks 96 busId 4000
e07r2n05:906:4284 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:906:4284 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n05:907:4286 [1] NCCL INFO Connected all trees comm 0x2af92c000ab0 nRanks 96 busId 26000
e07r2n05:907:4286 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n05:907:4286 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26995:30257 [0] NCCL INFO Connected all trees comm 0x2b7ef4000ab0 nRanks 96 busId 4000
e07r2n16:26995:30257 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n16:26995:30257 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n16:26996:30258 [1] NCCL INFO Connected all trees comm 0x2ae0fc000ab0 nRanks 96 busId 26000
e07r2n16:26996:30258 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n16:26996:30258 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28147:31332 [1] NCCL INFO Channel 00 : 69[26000] -> 68[4000] via P2P/IPC comm 0x2b1c40000ab0 nRanks 96
e07r2n15:28147:31332 [1] NCCL INFO Channel 01 : 69[26000] -> 68[4000] via P2P/IPC comm 0x2b1c40000ab0 nRanks 96
e07r2n15:28145:31330 [0] NCCL INFO Connected all trees comm 0x2b20a4000ab0 nRanks 96 busId 4000
e07r2n15:28145:31330 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28145:31330 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28148:31326 [2] NCCL INFO Connected all trees comm 0x2afcc0000ab0 nRanks 96 busId 43000
e07r2n15:28147:31332 [1] NCCL INFO Connected all trees comm 0x2b1c40000ab0 nRanks 96 busId 26000
e07r2n15:28148:31326 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28148:31326 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n15:28147:31332 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n15:28147:31332 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22275:25517 [0] NCCL INFO Connected all trees comm 0x2b6bd8000ab0 nRanks 96 busId 4000
e07r2n14:22275:25517 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n14:22275:25517 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22276:25518 [1] NCCL INFO Connected all trees comm 0x2af10c000ab0 nRanks 96 busId 26000
e07r2n14:22276:25518 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n14:22276:25518 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24490:27776 [0] NCCL INFO Channel 00/0 : 92[4000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b876c000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 00/0 : 60[4000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b858c000ab0 nRanks 96
e07r2n04:1826:5190 [0] NCCL INFO Channel 01/0 : 24[4000] -> 21[26000] [send] via NET/IB/0 comm 0x2b0c64000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Channel 01/0 : 84[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2b97a0000ab0 nRanks 96
e07r2n10:29369:32190 [0] NCCL INFO Channel 01/0 : 52[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2b2760000ab0 nRanks 96
e07r2n01:626:4329 [0] NCCL INFO Channel 00/0 : 12[4000] -> 8[4000] [send] via NET/IB/0 comm 0x2ae9b8000ab0 nRanks 96
e07r2n09:23023:26308 [0] NCCL INFO Channel 00/0 : 44[4000] -> 40[4000] [send] via NET/IB/0 comm 0x2af9d0000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 00/0 : 9[26000] -> 4[4000] [receive] via NET/IB/0 comm 0x2b7088000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 00/0 : 41[26000] -> 36[4000] [receive] via NET/IB/0 comm 0x2ac9e0000ab0 nRanks 96
e07r2n02:4759:8314 [2] NCCL INFO comm 0x2b3524000ab0 rank 18 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n02:4758:8319 [1] NCCL INFO comm 0x2b8694000ab0 rank 17 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n02:4760:8316 [3] NCCL INFO comm 0x2b7558000ab0 rank 19 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n17:31185:1575 [1] NCCL INFO comm 0x2ac95c000ab0 rank 77 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n17:31184:1572 [0] NCCL INFO comm 0x2bb19c000ab0 rank 76 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n17:31186:1568 [2] NCCL INFO comm 0x2b30ec000ab0 rank 78 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n17:31187:1570 [3] NCCL INFO comm 0x2b137c000ab0 rank 79 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n02:4756:8318 [0] NCCL INFO comm 0x2b71c4000ab0 rank 16 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r3n01:6046:9750 [0] NCCL INFO Connected all trees comm 0x2b0dd4000ab0 nRanks 96 busId 4000
e07r3n01:6046:9750 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6046:9750 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n01:6047:9752 [1] NCCL INFO Connected all trees comm 0x2ade98000ab0 nRanks 96 busId 26000
e07r3n01:6047:9752 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n01:6047:9752 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n13:31206:1486 [0] NCCL INFO Connected all trees comm 0x2aec38000ab0 nRanks 96 busId 4000
e07r2n13:31206:1486 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31206:1486 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n13:31207:1488 [1] NCCL INFO Connected all trees comm 0x2aedb4000ab0 nRanks 96 busId 26000
e07r2n13:31207:1488 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n13:31207:1488 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n04:1826:5190 [0] NCCL INFO Connected all trees comm 0x2b0c64000ab0 nRanks 96 busId 4000
e07r2n04:1826:5190 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n04:1826:5190 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n04:1827:5193 [1] NCCL INFO Connected all trees comm 0x2ac224000ab0 nRanks 96 busId 26000
e07r2n04:1827:5193 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n04:1827:5193 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n01:626:4329 [0] NCCL INFO Connected all trees comm 0x2ae9b8000ab0 nRanks 96 busId 4000
e07r2n01:626:4329 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n01:626:4329 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n01:627:4330 [1] NCCL INFO Connected all trees comm 0x2b7854000ab0 nRanks 96 busId 26000
e07r2n01:627:4330 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n01:627:4330 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n09:23023:26308 [0] NCCL INFO Connected all trees comm 0x2af9d0000ab0 nRanks 96 busId 4000
e07r2n09:23023:26308 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n09:23023:26308 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n09:23024:26305 [1] NCCL INFO Connected all trees comm 0x2b3c5c000ab0 nRanks 96 busId 26000
e07r2n03:21864:25088 [1] NCCL INFO Channel 00 : 21[26000] -> 20[4000] via P2P/IPC comm 0x2b34e0000ab0 nRanks 96
e07r2n09:23024:26305 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n09:23024:26305 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n03:21864:25088 [1] NCCL INFO Channel 01 : 21[26000] -> 20[4000] via P2P/IPC comm 0x2b34e0000ab0 nRanks 96
e07r2n18:25917:28643 [0] NCCL INFO Connected all trees comm 0x2b97a0000ab0 nRanks 96 busId 4000
e07r2n18:25917:28643 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n18:25917:28643 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n18:25918:28644 [1] NCCL INFO Connected all trees comm 0x2b0bb8000ab0 nRanks 96 busId 26000
e07r2n18:25918:28644 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n18:25918:28644 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n03:21863:25090 [0] NCCL INFO Connected all trees comm 0x2b3d8c000ab0 nRanks 96 busId 4000
e07r2n08:30378:1329 [1] NCCL INFO Channel 00 : 41[26000] -> 40[4000] via P2P/IPC comm 0x2b2e18000ab0 nRanks 96
e07r2n03:21863:25090 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n03:21863:25090 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n08:30378:1329 [1] NCCL INFO Channel 01 : 41[26000] -> 40[4000] via P2P/IPC comm 0x2b2e18000ab0 nRanks 96
e07r2n03:21864:25088 [1] NCCL INFO Connected all trees comm 0x2b34e0000ab0 nRanks 96 busId 26000
e07r2n03:21865:25084 [2] NCCL INFO Connected all trees comm 0x2b628c000ab0 nRanks 96 busId 43000
e07r2n03:21864:25088 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n03:21864:25088 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n03:21865:25084 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n03:21865:25084 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n10:29369:32190 [0] NCCL INFO Connected all trees comm 0x2b2760000ab0 nRanks 96 busId 4000
e07r2n10:29369:32190 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n10:29369:32190 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n10:29371:32191 [1] NCCL INFO Connected all trees comm 0x2ad734000ab0 nRanks 96 busId 26000
e07r2n10:29371:32191 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n10:29371:32191 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23101:26354 [1] NCCL INFO Channel 00 : 9[26000] -> 8[4000] via P2P/IPC comm 0x2ac104000ab0 nRanks 96
e07r2n00:23101:26354 [1] NCCL INFO Channel 01 : 9[26000] -> 8[4000] via P2P/IPC comm 0x2ac104000ab0 nRanks 96
e07r2n08:30379:1323 [2] NCCL INFO Connected all trees comm 0x2adfbc000ab0 nRanks 96 busId 43000
e07r2n08:30379:1323 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n08:30379:1323 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23102:26350 [2] NCCL INFO Connected all trees comm 0x2ace64000ab0 nRanks 96 busId 43000
e07r2n00:23102:26350 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23102:26350 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24490:27776 [0] NCCL INFO Channel 01/0 : 88[4000] -> 85[26000] [send] via NET/IB/0 comm 0x2b876c000ab0 nRanks 96
e07r2n12:16035:18767 [0] NCCL INFO Channel 01/0 : 56[4000] -> 53[26000] [send] via NET/IB/0 comm 0x2b858c000ab0 nRanks 96
e07r2n00:23099:26356 [0] NCCL INFO Channel 01/0 : 8[4000] -> 5[26000] [send] via NET/IB/0 comm 0x2b7e4c000ab0 nRanks 96
e07r2n08:30377:1327 [0] NCCL INFO Channel 01/0 : 40[4000] -> 37[26000] [send] via NET/IB/0 comm 0x2b4274000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Channel 01/0 : 36[4000] -> 32[4000] [send] via NET/IB/0 comm 0x2ac9e0000ab0 nRanks 96
e07r1n19:30599:1037 [0] NCCL INFO Channel 01/0 : 4[4000] -> 0[4000] [send] via NET/IB/0 comm 0x2b7088000ab0 nRanks 96
e07r2n16:26997:30254 [2] NCCL INFO comm 0x2b23d4000ab0 rank 74 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n05:908:4281 [2] NCCL INFO comm 0x2b03d0000ab0 rank 30 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n15:28147:31332 [1] NCCL INFO comm 0x2b1c40000ab0 rank 69 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n15:28150:31327 [3] NCCL INFO comm 0x2b3cb0000ab0 rank 71 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n15:28145:31330 [0] NCCL INFO comm 0x2b20a4000ab0 rank 68 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n14:22278:25514 [3] NCCL INFO comm 0x2b1220000ab0 rank 67 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n14:22275:25517 [0] NCCL INFO comm 0x2b6bd8000ab0 rank 64 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n05:907:4286 [1] NCCL INFO comm 0x2af92c000ab0 rank 29 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n05:909:4280 [3] NCCL INFO comm 0x2ac550000ab0 rank 31 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n05:906:4284 [0] NCCL INFO comm 0x2ba624000ab0 rank 28 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n16:26995:30257 [0] NCCL INFO comm 0x2b7ef4000ab0 rank 72 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n16:26998:30253 [3] NCCL INFO comm 0x2b5f04000ab0 rank 75 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n15:28148:31326 [2] NCCL INFO comm 0x2afcc0000ab0 rank 70 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n16:26996:30258 [1] NCCL INFO comm 0x2ae0fc000ab0 rank 73 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n00:24490:27776 [0] NCCL INFO Connected all trees comm 0x2b876c000ab0 nRanks 96 busId 4000
e07r3n00:24490:27776 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r3n00:24490:27776 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n00:24491:27778 [1] NCCL INFO Connected all trees comm 0x2b9dcc000ab0 nRanks 96 busId 26000
e07r3n00:24491:27778 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n12:16035:18767 [0] NCCL INFO Connected all trees comm 0x2b858c000ab0 nRanks 96 busId 4000
e07r3n00:24491:27778 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n12:16035:18767 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n12:16035:18767 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n12:16037:18770 [1] NCCL INFO Connected all trees comm 0x2ae364000ab0 nRanks 96 busId 26000
e07r2n12:16037:18770 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n12:16037:18770 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n14:22277:25513 [2] NCCL INFO comm 0x2b75dc000ab0 rank 66 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n19:2109:5605 [1] NCCL INFO Channel 00 : 85[26000] -> 84[4000] via P2P/IPC comm 0x2b3080000ab0 nRanks 96
e07r2n19:2109:5605 [1] NCCL INFO Channel 01 : 85[26000] -> 84[4000] via P2P/IPC comm 0x2b3080000ab0 nRanks 96
e07r2n14:22276:25518 [1] NCCL INFO comm 0x2af10c000ab0 rank 65 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n11:16527:19776 [1] NCCL INFO Channel 00 : 53[26000] -> 52[4000] via P2P/IPC comm 0x2aebfc000ab0 nRanks 96
e07r2n11:16527:19776 [1] NCCL INFO Channel 01 : 53[26000] -> 52[4000] via P2P/IPC comm 0x2aebfc000ab0 nRanks 96
e07r2n19:2107:5602 [0] NCCL INFO Connected all trees comm 0x2b5ad0000ab0 nRanks 96 busId 4000
e07r2n19:2107:5602 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n19:2107:5602 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n19:2109:5605 [1] NCCL INFO Connected all trees comm 0x2b3080000ab0 nRanks 96 busId 26000
e07r2n19:2110:5599 [2] NCCL INFO Connected all trees comm 0x2b9f58000ab0 nRanks 96 busId 43000
e07r2n19:2109:5605 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n19:2109:5605 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n19:2110:5599 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n19:2110:5599 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16526:19775 [0] NCCL INFO Connected all trees comm 0x2b86d4000ab0 nRanks 96 busId 4000
e07r2n11:16526:19775 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16526:19775 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16527:19776 [1] NCCL INFO Connected all trees comm 0x2aebfc000ab0 nRanks 96 busId 26000
e07r2n11:16528:19774 [2] NCCL INFO Connected all trees comm 0x2b8cf0000ab0 nRanks 96 busId 43000
e07r2n11:16527:19776 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16527:19776 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n11:16528:19774 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n11:16528:19774 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23099:26356 [0] NCCL INFO Connected all trees comm 0x2b7e4c000ab0 nRanks 96 busId 4000
e07r2n00:23099:26356 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23099:26356 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n00:23101:26354 [1] NCCL INFO Connected all trees comm 0x2ac104000ab0 nRanks 96 busId 26000
e07r2n00:23101:26354 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n00:23101:26354 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n08:30377:1327 [0] NCCL INFO Connected all trees comm 0x2b4274000ab0 nRanks 96 busId 4000
e07r2n08:30377:1327 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n08:30377:1327 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n08:30378:1329 [1] NCCL INFO Connected all trees comm 0x2b2e18000ab0 nRanks 96 busId 26000
e07r2n08:30378:1329 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n08:30378:1329 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n07:11608:15052 [1] NCCL INFO Channel 00 : 37[26000] -> 36[4000] via P2P/IPC comm 0x2aed18000ab0 nRanks 96
e07r2n07:11608:15052 [1] NCCL INFO Channel 01 : 37[26000] -> 36[4000] via P2P/IPC comm 0x2aed18000ab0 nRanks 96
e07r1n19:30600:1040 [1] NCCL INFO Channel 00 : 5[26000] -> 4[4000] via P2P/IPC comm 0x2b48a0000ab0 nRanks 96
e07r1n19:30600:1040 [1] NCCL INFO Channel 01 : 5[26000] -> 4[4000] via P2P/IPC comm 0x2b48a0000ab0 nRanks 96
e07r2n07:11607:15053 [0] NCCL INFO Connected all trees comm 0x2ac9e0000ab0 nRanks 96 busId 4000
e07r2n07:11607:15053 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11607:15053 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n19:30599:1037 [0] NCCL INFO Connected all trees comm 0x2b7088000ab0 nRanks 96 busId 4000
e07r1n19:30599:1037 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n19:30599:1037 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n07:11608:15052 [1] NCCL INFO Connected all trees comm 0x2aed18000ab0 nRanks 96 busId 26000
e07r2n07:11608:15052 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11608:15052 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n19:30601:1035 [2] NCCL INFO Connected all trees comm 0x2b9774000ab0 nRanks 96 busId 43000
e07r1n19:30600:1040 [1] NCCL INFO Connected all trees comm 0x2b48a0000ab0 nRanks 96 busId 26000
e07r1n19:30601:1035 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n19:30601:1035 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n19:30600:1040 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n19:30600:1040 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n07:11609:15048 [2] NCCL INFO Connected all trees comm 0x2accec000ab0 nRanks 96 busId 43000
e07r2n07:11609:15048 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n07:11609:15048 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n06:26608:29334 [0] NCCL INFO Connected all trees comm 0x2b2c04000ab0 nRanks 96 busId 4000
e07r2n06:26608:29334 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n06:26608:29334 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r2n06:26609:29335 [1] NCCL INFO Connected all trees comm 0x2ae97c000ab0 nRanks 96 busId 26000
e07r2n06:26609:29335 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r2n06:26609:29335 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n18:15185:19054 [0] NCCL INFO Connected all trees comm 0x2b2ef8000ab0 nRanks 96 busId 4000
e07r1n18:15185:19054 [0] NCCL INFO Using tuning table 0 with LL128 disabled
e07r1n18:15185:19054 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n18:15185:19054 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r1n18:15187:19060 [1] NCCL INFO Connected all trees comm 0x2ac9ac000ab0 nRanks 96 busId 26000
e07r1n18:15187:19060 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e07r1n18:15187:19060 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e07r3n01:6048:9746 [2] NCCL INFO comm 0x2b4198000ab0 rank 94 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n04:1828:5186 [2] NCCL INFO comm 0x2b227c000ab0 rank 26 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n01:6046:9750 [0] NCCL INFO comm 0x2b0dd4000ab0 rank 92 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n04:1829:5188 [3] NCCL INFO comm 0x2b68a8000ab0 rank 27 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n09:23026:26303 [3] NCCL INFO comm 0x2b4b4c000ab0 rank 47 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n09:23024:26305 [1] NCCL INFO comm 0x2b3c5c000ab0 rank 45 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n18:25919:28639 [2] NCCL INFO comm 0x2af1a8000ab0 rank 82 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n18:25917:28643 [0] NCCL INFO comm 0x2b97a0000ab0 rank 80 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n03:21864:25088 [1] NCCL INFO comm 0x2b34e0000ab0 rank 21 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n10:29369:32190 [0] NCCL INFO comm 0x2b2760000ab0 rank 48 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r3n01:6049:9748 [3] NCCL INFO comm 0x2b508c000ab0 rank 95 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n10:29372:32186 [2] NCCL INFO comm 0x2b4970000ab0 rank 50 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n01:6047:9752 [1] NCCL INFO comm 0x2ade98000ab0 rank 93 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n13:31209:1487 [2] NCCL INFO comm 0x2aba34000ab0 rank 62 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n13:31206:1486 [0] NCCL INFO comm 0x2aec38000ab0 rank 60 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n13:31210:1485 [3] NCCL INFO comm 0x2afab0000ab0 rank 63 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n04:1826:5190 [0] NCCL INFO comm 0x2b0c64000ab0 rank 24 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n04:1827:5193 [1] NCCL INFO comm 0x2ac224000ab0 rank 25 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n01:626:4329 [0] NCCL INFO comm 0x2ae9b8000ab0 rank 12 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n01:628:4325 [2] NCCL INFO comm 0x2aeb74000ab0 rank 14 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n01:627:4330 [1] NCCL INFO comm 0x2b7854000ab0 rank 13 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n01:629:4327 [3] NCCL INFO comm 0x2ad0f4000ab0 rank 15 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n09:23025:26302 [2] NCCL INFO comm 0x2b4748000ab0 rank 46 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n09:23023:26308 [0] NCCL INFO comm 0x2af9d0000ab0 rank 44 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n03:21866:25086 [3] NCCL INFO comm 0x2b6e40000ab0 rank 23 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n18:25918:28644 [1] NCCL INFO comm 0x2b0bb8000ab0 rank 81 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n18:25920:28641 [3] NCCL INFO comm 0x2af828000ab0 rank 83 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n03:21865:25084 [2] NCCL INFO comm 0x2b628c000ab0 rank 22 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n03:21863:25090 [0] NCCL INFO comm 0x2b3d8c000ab0 rank 20 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n10:29371:32191 [1] NCCL INFO comm 0x2ad734000ab0 rank 49 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n13:31207:1488 [1] NCCL INFO comm 0x2aedb4000ab0 rank 61 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n10:29373:32188 [3] NCCL INFO comm 0x2b2da4000ab0 rank 51 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r3n00:24491:27778 [1] NCCL INFO comm 0x2b9dcc000ab0 rank 89 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n00:24493:27774 [3] NCCL INFO comm 0x2b0888000ab0 rank 91 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n11:16529:19772 [3] NCCL INFO comm 0x2b84d8000ab0 rank 55 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n11:16526:19775 [0] NCCL INFO comm 0x2b86d4000ab0 rank 52 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n12:16035:18767 [0] NCCL INFO comm 0x2b858c000ab0 rank 56 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n11:16528:19774 [2] NCCL INFO comm 0x2b8cf0000ab0 rank 54 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n08:30377:1327 [0] NCCL INFO comm 0x2b4274000ab0 rank 40 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n00:23099:26356 [0] NCCL INFO comm 0x2b7e4c000ab0 rank 8 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n08:30380:1325 [3] NCCL INFO comm 0x2b8818000ab0 rank 43 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n08:30378:1329 [1] NCCL INFO comm 0x2b2e18000ab0 rank 41 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n12:16037:18770 [1] NCCL INFO comm 0x2ae364000ab0 rank 57 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r3n00:24490:27776 [0] NCCL INFO comm 0x2b876c000ab0 rank 88 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r3n00:24492:27773 [2] NCCL INFO comm 0x2b9934000ab0 rank 90 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n07:11609:15048 [2] NCCL INFO comm 0x2accec000ab0 rank 38 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n12:16038:18764 [2] NCCL INFO comm 0x2b9664000ab0 rank 58 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n06:26610:29330 [2] NCCL INFO comm 0x2b9594000ab0 rank 34 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n12:16039:18766 [3] NCCL INFO comm 0x2b4134000ab0 rank 59 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n18:15188:19056 [2] NCCL INFO comm 0x2b6c7c000ab0 rank 2 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n19:2110:5599 [2] NCCL INFO comm 0x2b9f58000ab0 rank 86 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n06:26609:29335 [1] NCCL INFO comm 0x2ae97c000ab0 rank 33 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n00:23102:26350 [2] NCCL INFO comm 0x2ace64000ab0 rank 10 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n08:30379:1323 [2] NCCL INFO comm 0x2adfbc000ab0 rank 42 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r1n18:15187:19060 [1] NCCL INFO comm 0x2ac9ac000ab0 rank 1 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r1n18:15189:19057 [3] NCCL INFO comm 0x2b6fbc000ab0 rank 3 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n00:23101:26354 [1] NCCL INFO comm 0x2ac104000ab0 rank 9 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n19:2111:5601 [3] NCCL INFO comm 0x2b3334000ab0 rank 87 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n19:2107:5602 [0] NCCL INFO comm 0x2b5ad0000ab0 rank 84 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n19:2109:5605 [1] NCCL INFO comm 0x2b3080000ab0 rank 85 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n07:11610:15050 [3] NCCL INFO comm 0x2affe4000ab0 rank 39 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n11:16527:19776 [1] NCCL INFO comm 0x2aebfc000ab0 rank 53 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n00:23103:26351 [3] NCCL INFO comm 0x2b8604000ab0 rank 11 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n19:30600:1040 [1] NCCL INFO comm 0x2b48a0000ab0 rank 5 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r1n19:30602:1034 [3] NCCL INFO comm 0x2b2e08000ab0 rank 7 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n07:11608:15052 [1] NCCL INFO comm 0x2aed18000ab0 rank 37 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r1n19:30599:1037 [0] NCCL INFO comm 0x2b7088000ab0 rank 4 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n07:11607:15053 [0] NCCL INFO comm 0x2ac9e0000ab0 rank 36 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n19:30601:1035 [2] NCCL INFO comm 0x2b9774000ab0 rank 6 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e07r2n06:26608:29334 [0] NCCL INFO comm 0x2b2c04000ab0 rank 32 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r1n18:15185:19054 [0] NCCL INFO comm 0x2b2ef8000ab0 rank 0 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e07r2n06:26611:29331 [3] NCCL INFO comm 0x2b62e4000ab0 rank 35 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 2.9718, 'learning_rate': 2e-05, 'epoch': 0.02}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}
{'loss': 3.101, 'learning_rate': 2e-05, 'epoch': 0.03}