START TIME: Fri Oct 27 11:24:49 CST 2023
e08r2n08
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
LRANK===============================2
LRANK===============================2
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================2
LRANK===============================0
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
LRANK===============================2
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================1
WORLD_SIZE*************96
LRANK===============================2
LRANK===============================2
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================2
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
LRANK===============================3
LRANK===============================3
WORLD_SIZE*************96
WORLD_SIZE*************96
LRANK===============================3
WORLD_SIZE*************96
[2023-10-27 11:25:35,734] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,734] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,729] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,725] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,734] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,732] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,730] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,734] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,729] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,726] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,734] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,730] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,734] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,729] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,726] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,730] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,729] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,735] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,733] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,733] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,739] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,739] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,732] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,730] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,739] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,736] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,727] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,733] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,733] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,731] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,739] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,731] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,739] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,732] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,734] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,739] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,737] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,738] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,733] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,740] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,732] [INFO] [comm.py:606:init_distributed] Not using the DeepSpeed or dist launchers, attempting to detect MPI environment...
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=0, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:622:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=20, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=24, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=32, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,993] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=8, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=12, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=1, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=21, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=25, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=33, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,993] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=9, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=13, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,991] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=36, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=4, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=64, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=52, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=68, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,991] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=60, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=72, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,988] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=88, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=40, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=84, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=80, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=16, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,994] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=92, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=28, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=44, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=76, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,992] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=56, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=48, local_rank=0, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,992] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=37, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=5, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=65, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=53, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=69, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,991] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=61, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=73, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,988] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=89, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=41, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=85, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=81, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=17, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,994] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=93, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=29, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=45, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=77, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,992] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=57, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,997] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=49, local_rank=1, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=23, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=2, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=3, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=22, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,993] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=39, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=19, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=6, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,001] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=75, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=55, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=15, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,992] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=62, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,989] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=90, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=42, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,993] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=38, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=18, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=7, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=54, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=14, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,992] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=63, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,989] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=91, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=43, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,001] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=71, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,994] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=11, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=66, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,001] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=31, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,996] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=95, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=79, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=82, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=27, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,001] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=87, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=51, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=34, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=47, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,994] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=59, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,001] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=70, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,994] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=10, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=67, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,001] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=30, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,996] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=94, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,000] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=78, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=83, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=26, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,001] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=86, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,998] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=50, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=35, local_rank=3, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,999] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=46, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:35,994] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=58, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
[2023-10-27 11:25:36,001] [INFO] [comm.py:656:mpi_discovery] Discovered MPI settings of world_rank=74, local_rank=2, world_size=96, master_addr=10.5.8.29, master_port=29500
e08r2n08:11393:11393 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.29<0>
e08r2n08:11393:11393 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n08:11393:11393 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
RCCL version 2.13.4+hip5.4 HEAD:82f11f7
e08r2n08:11393:12596 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.29<0>
e08r2n08:11393:12596 [0] NCCL INFO Using network IB
e08r2n08:11395:11395 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.29<0>
e08r2n08:11395:11395 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n08:11395:11395 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n08:11396:11396 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.29<0>
e08r2n08:11396:11396 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n08:11396:11396 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n08:11397:11397 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.29<0>
e08r2n08:11397:11397 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n08:11397:11397 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n03:24232:24232 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.44<0>
e08r3n03:24232:24232 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n15:23877:23877 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.36<0>
e08r2n15:23877:23877 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n15:23879:23879 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.36<0>
e08r2n15:23879:23879 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n15:23877:23877 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n10:2340:2340 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.31<0>
e08r2n10:2340:2340 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n15:23879:23879 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n10:2340:2340 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n11:31583:31583 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.52<0>
e08r3n11:31583:31583 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n07:20445:20445 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.48<0>
e08r3n07:20445:20445 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n03:24231:24231 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.44<0>
e08r3n03:24231:24231 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n19:10428:10428 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.40<0>
e08r2n19:10428:10428 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n11:31585:31585 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.52<0>
e08r3n11:31585:31585 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n10:2344:2344 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.31<0>
e08r2n10:2344:2344 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n15:23880:23880 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.36<0>
e08r2n15:23880:23880 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n11:31583:31583 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n11:31585:31585 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n10:2344:2344 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n15:23880:23880 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n07:20445:20445 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n19:10427:10427 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.40<0>
e08r2n19:10427:10427 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n02:14034:14034 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.43<0>
e08r3n02:14034:14034 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n02:14036:14036 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.43<0>
e08r3n02:14036:14036 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n03:24232:24232 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n03:24231:24231 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n02:14034:14034 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n02:14036:14036 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n11:31586:31586 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.52<0>
e08r3n11:31586:31586 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n15:23881:23881 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.36<0>
e08r2n14:20092:20092 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.35<0>
e08r2n14:20092:20092 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n15:23881:23881 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n11:31586:31586 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n07:20449:20449 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.48<0>
e08r3n07:20449:20449 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n19:10428:10428 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n02:14038:14038 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.43<0>
e08r3n02:14038:14038 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n15:23881:23881 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n07:20449:20449 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n19:10427:10427 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n02:14038:14038 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n19:10429:10429 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.40<0>
e08r2n19:10429:10429 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n18:2636:2636 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.39<0>
e08r2n18:2636:2636 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n11:31587:31587 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.52<0>
e08r3n11:31587:31587 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n14:20092:20092 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n19:10429:10429 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n11:31587:31587 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n18:2634:2634 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.39<0>
e08r2n18:2634:2634 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n18:2636:2636 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n18:2634:2634 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n05:26315:26315 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.46<0>
e08r3n05:26315:26315 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n16:15463:15463 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.37<0>
e08r2n16:15463:15463 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n08:14730:14730 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.49<0>
e08r3n08:14731:14731 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.49<0>
e08r3n08:14731:14731 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n08:14730:14730 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n02:14037:14037 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.43<0>
e08r3n05:26317:26317 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.46<0>
e08r3n02:14037:14037 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n05:26317:26317 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n19:10430:10430 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.40<0>
e08r2n19:10430:10430 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n12:10508:10508 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.33<0>
e08r2n12:10508:10508 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n05:26315:26315 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n14:20094:20094 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.35<0>
e08r2n13:26093:26093 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.34<0>
e08r2n13:26093:26093 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n14:20094:20094 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n02:14037:14037 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n16:15463:15463 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n05:26317:26317 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n19:10430:10430 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n10:20256:20256 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.51<0>
e08r3n10:20256:20256 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n01:21692:21692 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.42<0>
e08r3n01:21692:21692 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n08:14731:14731 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n14:20094:20094 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n09:28729:28729 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.30<0>
e08r2n09:28729:28729 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n09:28728:28728 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.30<0>
e08r2n09:28728:28728 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n16:15467:15467 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.37<0>
e08r2n16:15467:15467 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n08:14730:14730 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n13:26093:26093 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n18:2637:2637 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.39<0>
e08r2n18:2637:2637 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n16:15467:15467 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n11:7501:7501 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.32<0>
e08r2n11:7501:7501 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n10:20256:20256 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n03:24233:24233 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.44<0>
e08r3n03:24233:24233 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n01:21692:21692 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n09:28729:28729 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n12:10509:10509 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.33<0>
e08r2n12:10509:10509 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n18:2637:2637 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n16:15465:15465 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.37<0>
e08r2n14:20096:20096 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.35<0>
e08r2n16:15465:15465 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n14:20096:20096 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n03:24233:24233 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n09:28728:28728 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n11:7502:7502 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.32<0>
e08r2n11:7502:7502 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n03:24229:24229 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.44<0>
e08r3n03:24229:24229 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n11:7501:7501 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n16:15465:15465 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n14:20096:20096 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n11:7502:7502 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n13:26095:26095 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.34<0>
e08r2n13:26095:26095 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n03:24229:24229 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n10:20258:20258 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.51<0>
e08r3n10:20258:20258 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n01:21696:21696 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.42<0>
e08r3n01:21696:21696 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n13:26095:26095 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n10:20258:20258 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n05:26318:26318 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.46<0>
e08r3n01:21696:21696 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n05:26318:26318 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n13:26094:26094 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.34<0>
e08r2n13:26094:26094 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n01:21694:21694 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.42<0>
e08r3n01:21694:21694 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n13:26094:26094 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n05:26318:26318 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n14:20095:20095 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.35<0>
e08r2n14:20095:20095 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n11:7503:7503 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.32<0>
e08r2n11:7503:7503 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n01:21695:21695 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.42<0>
e08r3n01:21695:21695 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n09:28731:28731 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.30<0>
e08r3n01:21694:21694 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n09:28731:28731 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n12:10511:10511 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.33<0>
e08r2n10:2342:2342 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.31<0>
e08r2n12:10511:10511 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n10:2342:2342 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n14:20095:20095 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n11:7503:7503 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n18:2638:2638 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.39<0>
e08r2n18:2638:2638 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n01:21695:21695 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n09:28731:28731 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n10:2342:2342 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n16:15466:15466 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.37<0>
e08r2n16:15466:15466 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n05:26319:26319 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.46<0>
e08r3n05:26319:26319 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n18:2638:2638 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n16:15466:15466 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n05:26319:26319 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n00:29325:29325 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.41<0>
e08r3n00:29325:29325 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n17:22820:22820 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.38<0>
e08r2n17:22820:22820 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n00:29327:29327 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.41<0>
e08r3n00:29327:29327 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n10:2343:2343 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.31<0>
e08r2n10:2343:2343 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n09:28730:28730 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.30<0>
e08r2n09:28730:28730 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n12:10510:10510 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.33<0>
e08r2n12:10510:10510 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n10:20259:20259 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.51<0>
e08r3n10:20259:20259 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n10:2343:2343 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n09:28730:28730 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n00:29325:29325 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n07:20447:20447 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.48<0>
e08r3n10:20259:20259 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n07:20447:20447 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n17:22824:22824 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.38<0>
e08r2n17:22824:22824 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n11:7504:7504 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.32<0>
e08r2n11:7504:7504 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n00:29327:29327 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n07:20447:20447 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n04:31686:31686 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.45<0>
e08r3n04:31686:31686 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n11:7504:7504 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n13:26096:26096 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.34<0>
e08r2n13:26096:26096 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n10:20260:20260 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.51<0>
e08r3n10:20260:20260 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n09:10072:10072 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.50<0>
e08r3n09:10072:10072 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n13:26096:26096 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n07:20448:20448 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.48<0>
e08r3n07:20448:20448 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n09:10076:10076 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.50<0>
e08r3n10:20260:20260 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n09:10074:10074 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.50<0>
e08r3n09:10076:10076 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n09:10074:10074 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n07:20448:20448 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n00:29328:29328 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.41<0>
e08r3n00:29330:29330 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.41<0>
e08r3n09:10072:10072 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n00:29328:29328 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n09:10074:10074 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n00:29330:29330 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n09:10076:10076 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n09:10075:10075 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.50<0>
e08r3n09:10075:10075 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n00:29328:29328 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n00:29330:29330 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n06:24081:24081 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.47<0>
e08r3n06:24081:24081 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n09:10075:10075 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n17:22822:22822 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.38<0>
e08r2n17:22822:22822 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n08:14729:14729 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.49<0>
e08r3n08:14729:14729 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n17:22820:22820 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n08:14729:14729 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n04:31686:31686 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n04:31688:31688 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.45<0>
e08r3n04:31688:31688 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n17:22824:22824 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n06:24085:24085 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.47<0>
e08r3n06:24085:24085 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n04:31688:31688 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n17:22822:22822 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n06:24081:24081 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n06:24085:24085 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n06:24083:24083 [1] NCCL INFO Bootstrap : Using ib0:11.5.8.47<0>
e08r2n17:22823:22823 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.38<0>
e08r3n06:24083:24083 [1] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r2n17:22823:22823 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n06:24083:24083 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n17:22823:22823 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n06:24084:24084 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.47<0>
e08r3n06:24084:24084 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n06:24084:24084 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n04:31690:31690 [3] NCCL INFO Bootstrap : Using ib0:11.5.8.45<0>
e08r3n04:31690:31690 [3] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n04:31690:31690 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n04:31689:31689 [2] NCCL INFO Bootstrap : Using ib0:11.5.8.45<0>
e08r3n04:31689:31689 [2] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n04:31689:31689 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r3n08:14727:14727 [0] NCCL INFO Bootstrap : Using ib0:11.5.8.49<0>
e08r3n08:14727:14727 [0] NCCL INFO Plugin name set by env to librccl-net-none.so
e08r3n08:14727:14727 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n12:10509:10509 [1] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n12:10511:10511 [3] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n12:10510:10510 [2] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n12:10508:10508 [0] NCCL INFO NET/Plugin : No plugin found (librccl-net-none.so), using internal implementation
e08r2n08:11395:12832 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.29<0>
e08r2n08:11395:12832 [1] NCCL INFO Using network IB
e08r2n08:11397:12837 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.29<0>
e08r2n08:11397:12837 [3] NCCL INFO Using network IB
e08r2n08:11396:12836 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.29<0>
e08r2n08:11396:12836 [2] NCCL INFO Using network IB
e08r2n15:23879:25082 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.36<0>
e08r2n15:23879:25082 [1] NCCL INFO Using network IB
e08r2n15:23877:25081 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.36<0>
e08r2n15:23877:25081 [0] NCCL INFO Using network IB
e08r2n15:23880:25090 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.36<0>
e08r2n15:23880:25090 [2] NCCL INFO Using network IB
e08r3n11:31583:32648 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.52<0>
e08r3n11:31583:32648 [0] NCCL INFO Using network IB
e08r3n11:31585:32654 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.52<0>
e08r3n11:31585:32654 [1] NCCL INFO Using network IB
e08r2n15:23881:25092 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.36<0>
e08r2n15:23881:25092 [3] NCCL INFO Using network IB
e08r3n11:31587:32657 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.52<0>
e08r3n11:31587:32657 [3] NCCL INFO Using network IB
e08r2n10:2340:3530 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.31<0>
e08r2n10:2340:3530 [0] NCCL INFO Using network IB
e08r2n10:2344:3537 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.31<0>
e08r2n10:2344:3537 [3] NCCL INFO Using network IB
e08r3n11:31586:32658 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.52<0>
e08r3n11:31586:32658 [2] NCCL INFO Using network IB
e08r3n07:20449:21403 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.48<0>
e08r3n07:20449:21403 [3] NCCL INFO Using network IB
e08r3n07:20445:21399 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.48<0>
e08r3n07:20445:21399 [0] NCCL INFO Using network IB
e08r3n02:14034:15017 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.43<0>
e08r3n02:14034:15017 [0] NCCL INFO Using network IB
e08r3n02:14036:15025 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.43<0>
e08r3n02:14036:15025 [1] NCCL INFO Using network IB
e08r3n02:14038:15023 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.43<0>
e08r3n02:14038:15023 [3] NCCL INFO Using network IB
e08r2n19:10429:11597 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.40<0>
e08r2n19:10429:11597 [2] NCCL INFO Using network IB
e08r3n03:24231:25246 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.44<0>
e08r3n03:24231:25246 [1] NCCL INFO Using network IB
e08r3n03:24233:25253 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.44<0>
e08r3n03:24233:25253 [3] NCCL INFO Using network IB
e08r3n03:24232:25250 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.44<0>
e08r3n03:24232:25250 [2] NCCL INFO Using network IB
e08r3n03:24229:25255 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.44<0>
e08r3n03:24229:25255 [0] NCCL INFO Using network IB
e08r2n18:2634:3704 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.39<0>
e08r2n18:2634:3704 [0] NCCL INFO Using network IB
e08r2n19:10428:11592 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.40<0>
e08r2n19:10428:11592 [1] NCCL INFO Using network IB
e08r2n19:10427:11596 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.40<0>
e08r2n19:10427:11596 [0] NCCL INFO Using network IB
e08r3n01:21692:22698 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.42<0>
e08r3n01:21692:22698 [0] NCCL INFO Using network IB
e08r2n19:10430:11601 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.40<0>
e08r2n19:10430:11601 [3] NCCL INFO Using network IB
e08r3n01:21696:22700 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.42<0>
e08r3n01:21696:22700 [3] NCCL INFO Using network IB
e08r2n14:20096:21173 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.35<0>
e08r2n14:20096:21173 [3] NCCL INFO Using network IB
e08r2n14:20092:21169 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.35<0>
e08r2n14:20092:21169 [0] NCCL INFO Using network IB
e08r2n14:20094:21174 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.35<0>
e08r2n14:20094:21174 [1] NCCL INFO Using network IB
e08r3n02:14037:15028 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.43<0>
e08r3n02:14037:15028 [2] NCCL INFO Using network IB
e08r2n13:26093:27103 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.34<0>
e08r2n13:26093:27103 [0] NCCL INFO Using network IB
e08r3n08:14730:15934 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.49<0>
e08r3n08:14730:15934 [2] NCCL INFO Using network IB
e08r3n08:14731:15932 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.49<0>
e08r3n08:14731:15932 [3] NCCL INFO Using network IB
e08r2n16:15465:16568 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.37<0>
e08r2n16:15465:16568 [1] NCCL INFO Using network IB
e08r2n18:2637:3714 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.39<0>
e08r2n18:2637:3714 [2] NCCL INFO Using network IB
e08r2n16:15467:16571 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.37<0>
e08r2n16:15467:16571 [3] NCCL INFO Using network IB
e08r3n01:21694:22707 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.42<0>
e08r3n01:21694:22707 [1] NCCL INFO Using network IB
e08r2n18:2636:3716 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.39<0>
e08r2n18:2636:3716 [1] NCCL INFO Using network IB
e08r2n16:15463:16564 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.37<0>
e08r2n16:15463:16564 [0] NCCL INFO Using network IB
e08r3n05:26315:27340 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.46<0>
e08r3n05:26315:27340 [0] NCCL INFO Using network IB
e08r3n10:20258:21448 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.51<0>
e08r3n10:20258:21448 [1] NCCL INFO Using network IB
e08r3n05:26317:27342 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.46<0>
e08r3n05:26317:27342 [1] NCCL INFO Using network IB
e08r3n10:20256:21447 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.51<0>
e08r3n10:20256:21447 [0] NCCL INFO Using network IB
e08r2n18:2638:3717 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.39<0>
e08r2n18:2638:3717 [3] NCCL INFO Using network IB
e08r3n05:26319:27347 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.46<0>
e08r3n05:26319:27347 [3] NCCL INFO Using network IB
e08r2n13:26095:27111 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.34<0>
e08r2n13:26095:27111 [2] NCCL INFO Using network IB
e08r2n13:26094:27113 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.34<0>
e08r2n13:26094:27113 [1] NCCL INFO Using network IB
e08r2n11:7501:8711 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.32<0>
e08r2n11:7501:8711 [0] NCCL INFO Using network IB
e08r2n11:7503:8719 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.32<0>
e08r2n11:7503:8719 [2] NCCL INFO Using network IB
e08r2n11:7502:8716 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.32<0>
e08r2n11:7502:8716 [1] NCCL INFO Using network IB
e08r3n10:20260:21455 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.51<0>
e08r3n10:20260:21455 [3] NCCL INFO Using network IB
e08r2n09:28728:29878 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.30<0>
e08r2n09:28728:29878 [0] NCCL INFO Using network IB
e08r2n09:28729:29883 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.30<0>
e08r2n09:28729:29883 [1] NCCL INFO Using network IB
e08r2n11:7504:8721 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.32<0>
e08r2n11:7504:8721 [3] NCCL INFO Using network IB
e08r2n09:28731:29886 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.30<0>
e08r2n09:28731:29886 [3] NCCL INFO Using network IB
e08r3n01:21695:22709 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.42<0>
e08r3n01:21695:22709 [2] NCCL INFO Using network IB
e08r2n14:20095:21178 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.35<0>
e08r2n14:20095:21178 [2] NCCL INFO Using network IB
e08r3n07:20447:21419 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.48<0>
e08r3n07:20447:21419 [1] NCCL INFO Using network IB
e08r3n05:26318:27350 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.46<0>
e08r3n05:26318:27350 [2] NCCL INFO Using network IB
e08r3n07:20448:21421 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.48<0>
e08r3n07:20448:21421 [2] NCCL INFO Using network IB
e08r2n13:26096:27117 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.34<0>
e08r2n13:26096:27117 [3] NCCL INFO Using network IB
e08r2n16:15466:16578 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.37<0>
e08r2n16:15466:16578 [2] NCCL INFO Using network IB
e08r2n09:28730:29888 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.30<0>
e08r2n09:28730:29888 [2] NCCL INFO Using network IB
e08r3n10:20259:21460 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.51<0>
e08r3n10:20259:21460 [2] NCCL INFO Using network IB
e08r2n10:2342:3554 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.31<0>
e08r2n10:2342:3554 [1] NCCL INFO Using network IB
e08r3n08:14729:15948 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.49<0>
e08r3n08:14729:15948 [1] NCCL INFO Using network IB
e08r3n00:29325:30324 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.41<0>
e08r3n00:29325:30324 [0] NCCL INFO Using network IB
e08r3n00:29328:30332 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.41<0>
e08r3n00:29328:30332 [2] NCCL INFO Using network IB
e08r3n00:29330:30333 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.41<0>
e08r3n00:29330:30333 [3] NCCL INFO Using network IB
e08r3n00:29327:30330 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.41<0>
e08r3n00:29327:30330 [1] NCCL INFO Using network IB
e08r2n10:2343:3556 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.31<0>
e08r2n10:2343:3556 [2] NCCL INFO Using network IB
e08r3n09:10072:11133 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.50<0>
e08r3n09:10072:11133 [0] NCCL INFO Using network IB
e08r3n09:10074:11130 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.50<0>
e08r3n09:10074:11130 [1] NCCL INFO Using network IB
e08r3n09:10076:11135 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.50<0>
e08r3n09:10076:11135 [3] NCCL INFO Using network IB
e08r3n09:10075:11136 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.50<0>
e08r3n09:10075:11136 [2] NCCL INFO Using network IB
e08r3n04:31690:302 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.45<0>
e08r3n04:31690:302 [3] NCCL INFO Using network IB
e08r3n04:31686:32764 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.45<0>
e08r3n04:31686:32764 [0] NCCL INFO Using network IB
e08r3n04:31688:304 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.45<0>
e08r3n04:31688:304 [1] NCCL INFO Using network IB
e08r3n08:14727:15954 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.49<0>
e08r3n08:14727:15954 [0] NCCL INFO Using network IB
e08r3n04:31689:307 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.45<0>
e08r3n04:31689:307 [2] NCCL INFO Using network IB
e08r3n06:24083:25278 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.47<0>
e08r3n06:24083:25278 [1] NCCL INFO Using network IB
e08r3n06:24085:25276 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.47<0>
e08r3n06:24085:25276 [3] NCCL INFO Using network IB
e08r3n06:24084:25277 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.47<0>
e08r3n06:24084:25277 [2] NCCL INFO Using network IB
e08r3n06:24081:25273 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.47<0>
e08r3n06:24081:25273 [0] NCCL INFO Using network IB
e08r2n17:22824:23810 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.38<0>
e08r2n17:22824:23810 [3] NCCL INFO Using network IB
e08r2n17:22823:23812 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.38<0>
e08r2n17:22823:23812 [2] NCCL INFO Using network IB
e08r2n17:22822:23809 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.38<0>
e08r2n17:22822:23809 [1] NCCL INFO Using network IB
e08r2n17:22820:23808 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.38<0>
e08r2n17:22820:23808 [0] NCCL INFO Using network IB
e08r2n12:10510:11624 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.33<0>
e08r2n12:10510:11624 [2] NCCL INFO Using network IB
e08r2n12:10508:11619 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.33<0>
e08r2n12:10508:11619 [0] NCCL INFO Using network IB
e08r2n12:10511:11620 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.33<0>
e08r2n12:10511:11620 [3] NCCL INFO Using network IB
e08r2n12:10509:11625 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [RO]; OOB ib0:11.5.8.33<0>
e08r2n12:10509:11625 [1] NCCL INFO Using network IB
e08r3n11:31585:32654 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n11:31583:32648 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n05:26319:27347 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n08:14730:15934 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n10:20260:21455 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n10:20259:21460 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n08:14731:15932 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n08:14729:15948 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n08:14727:15954 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n06:24083:25278 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n10:20258:21448 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n05:26317:27342 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n06:24085:25276 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n10:20256:21447 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n09:10072:11133 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n07:20447:21419 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n11:31587:32657 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n09:28728:29878 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n11:31586:32658 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n09:10074:11130 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n09:10075:11136 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n09:10076:11135 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n01:21692:22698 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n08:11397:12837 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n05:26318:27350 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n05:26315:27340 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n08:11395:12832 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n08:11393:12596 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n01:21694:22707 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n08:11396:12836 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n06:24081:25273 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n06:24084:25277 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n18:2638:3717 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n04:31686:32764 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n07:20449:21403 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n10:2342:3554 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n09:28729:29883 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n00:29325:30324 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n07:20445:21399 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n13:26096:27117 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n07:20448:21421 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n09:28731:29886 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n09:28730:29888 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n16:15465:16568 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n00:29327:30330 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n00:29328:30332 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n12:10509:11625 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n04:31688:304 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n04:31689:307 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n04:31690:302 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n01:21695:22709 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n13:26095:27111 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n01:21696:22700 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n10:2340:3530 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n00:29330:30333 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n10:2344:3537 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n10:2343:3556 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n19:10430:11601 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n16:15463:16564 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n11:7501:8711 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n12:10508:11619 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n14:20095:21178 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n19:10429:11597 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n13:26094:27113 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n13:26093:27103 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n12:10511:11620 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n12:10510:11624 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n11:7503:8719 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n11:7502:8716 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n16:15466:16578 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n11:7504:8721 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n17:22820:23808 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n14:20092:21169 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n14:20096:21173 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n16:15467:16571 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n18:2637:3714 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n14:20094:21174 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n19:10427:11596 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n17:22822:23809 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n19:10428:11592 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n18:2634:3704 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n17:22824:23810 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n17:22823:23812 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n18:2636:3716 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n15:23877:25081 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n15:23879:25082 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n15:23880:25090 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r2n15:23881:25092 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n02:14037:15028 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n02:14038:15023 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n02:14036:15025 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n02:14034:15017 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0
e08r3n01:21692:22698 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n08:11397:12837 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n09:28728:29878 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n07:20447:21419 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n09:10072:11133 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n04:31686:32764 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n07:20449:21403 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n06:24083:25278 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n14:20095:21178 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n04:31688:304 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n07:20448:21421 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n06:24081:25273 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n14:20092:21169 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n09:28729:29883 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n09:10074:11130 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n01:21694:22707 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n18:2638:3717 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n08:11393:12596 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n08:11395:12832 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n08:11396:12836 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n04:31690:302 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n07:20445:21399 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n06:24085:25276 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n14:20096:21173 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n09:28730:29888 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n09:10075:11136 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n01:21696:22700 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n18:2637:3714 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n08:14731:15932 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n00:29327:30330 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n13:26096:27117 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n10:2342:3554 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n11:7501:8711 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n08:11397:12837 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n04:31689:307 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n07:20447:21419 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n06:24084:25277 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n14:20094:21174 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n09:28731:29886 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n09:10076:11135 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n01:21695:22709 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n18:2634:3704 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n08:14730:15934 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n00:29330:30333 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n13:26095:27111 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n10:2340:3530 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n11:7503:8719 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n12:10509:11625 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n19:10430:11601 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n16:15465:16568 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n15:23877:25081 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n10:20259:21460 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n17:22820:23808 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n02:14038:15023 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n08:11393:12596 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n08:11395:12832 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n08:11396:12836 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n04:31686:32764 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n07:20449:21403 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n06:24083:25278 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n14:20095:21178 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n09:28728:29878 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n09:10072:11133 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n01:21692:22698 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n18:2636:3716 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n08:14727:15954 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n00:29325:30324 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n13:26094:27113 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n10:2344:3537 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n11:7504:8721 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n12:10508:11619 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n19:10429:11597 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n16:15463:16564 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n15:23879:25082 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n10:20260:21455 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n17:22822:23809 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n02:14037:15028 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n05:26319:27347 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n11:31585:32654 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n04:31688:304 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n07:20448:21421 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n06:24081:25273 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n14:20092:21169 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n09:28729:29883 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n09:10074:11130 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n01:21694:22707 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n18:2638:3717 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n08:14729:15948 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n00:29328:30332 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n13:26093:27103 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n10:2343:3556 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n11:7502:8716 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n12:10511:11620 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n19:10427:11596 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n16:15466:16578 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n15:23880:25090 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n10:20256:21447 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n17:22824:23810 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n02:14036:15025 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n05:26317:27342 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n11:31583:32648 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n04:31689:307 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n07:20445:21399 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n06:24085:25276 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n14:20094:21174 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n09:28730:29888 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n09:10075:11136 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n01:21696:22700 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n18:2634:3704 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n08:14731:15932 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n00:29327:30330 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n13:26096:27117 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n10:2342:3554 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n11:7501:8711 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n12:10510:11624 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n19:10428:11592 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n16:15467:16571 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n15:23881:25092 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n10:20258:21448 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n17:22823:23812 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n02:14034:15017 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n05:26315:27340 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n11:31587:32657 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n04:31690:302 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n06:24084:25277 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n14:20096:21173 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n09:28731:29886 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n09:10076:11135 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n01:21695:22709 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n18:2637:3714 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n08:14730:15934 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n00:29330:30333 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n13:26095:27111 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n10:2340:3530 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n11:7503:8719 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n12:10509:11625 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n19:10430:11601 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n16:15463:16564 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n15:23877:25081 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n10:20260:21455 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n17:22820:23808 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n02:14038:15023 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n05:26318:27350 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n11:31586:32658 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r2n18:2636:3716 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n08:14727:15954 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n00:29325:30324 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n13:26094:27113 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n10:2344:3537 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n11:7504:8721 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n12:10508:11619 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n19:10429:11597 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n16:15465:16568 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n15:23879:25082 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n10:20259:21460 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n17:22822:23809 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n02:14037:15028 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n05:26319:27347 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n11:31585:32654 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n08:14729:15948 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n00:29328:30332 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n13:26093:27103 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n10:2343:3556 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n11:7502:8716 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n12:10511:11620 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n19:10427:11596 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n16:15466:16578 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n15:23881:25092 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n10:20256:21447 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n17:22824:23810 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n02:14034:15017 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n05:26317:27342 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n11:31583:32648 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n12:10510:11624 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n19:10428:11592 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n16:15467:16571 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n15:23880:25090 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n10:20258:21448 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n17:22823:23812 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n02:14036:15025 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n05:26315:27340 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n11:31587:32657 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n05:26318:27350 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n11:31586:32658 [2] NCCL INFO Setting affinity for GPU 2 to ff0000

e08r3n03:24233:25253 [3] /data/jenkins_workspace/workspace/rccl_release/src/misc/rocm_smi_wrap.cc:38 NCCL WARN ROCm SMI init failure An error occurred during initialization, during monitor discovery or when when initializing internal data structures

e08r3n03:24231:25246 [1] /data/jenkins_workspace/workspace/rccl_release/src/misc/rocm_smi_wrap.cc:38 NCCL WARN ROCm SMI init failure An error occurred during initialization, during monitor discovery or when when initializing internal data structures

e08r3n03:24232:25250 [2] /data/jenkins_workspace/workspace/rccl_release/src/misc/rocm_smi_wrap.cc:38 NCCL WARN ROCm SMI init failure An error occurred during initialization, during monitor discovery or when when initializing internal data structures

e08r3n03:24229:25255 [0] /data/jenkins_workspace/workspace/rccl_release/src/misc/rocm_smi_wrap.cc:38 NCCL WARN ROCm SMI init failure An error occurred during initialization, during monitor discovery or when when initializing internal data structures
e08r3n03:24231:25246 [1] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n03:24233:25253 [3] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n03:24229:25255 [0] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n03:24232:25250 [2] NCCL INFO NCCL_P2P_LEVEL set by environment to SYS
e08r3n03:24231:25246 [1] NCCL INFO Setting affinity for GPU 0 to ff00
e08r3n03:24229:25255 [0] NCCL INFO Setting affinity for GPU 1052491776 to ff
e08r3n03:24233:25253 [3] NCCL INFO Setting affinity for GPU 0 to ff000000
e08r3n03:24232:25250 [2] NCCL INFO Setting affinity for GPU 0 to ff0000
e08r3n03:24233:25253 [3] NCCL INFO Trees [0] -1/-1/-1->63->62 [1] -1/-1/-1->63->62 comm 0x2b66e0000ab0 nRanks 96 busId 63000
e08r3n04:31686:32764 [0] NCCL INFO Trees [0] 65/80/-1->64->0 [1] 65/-1/-1->64->68 comm 0x2b08c0000ab0 nRanks 96 busId 4000
e08r3n04:31688:304 [1] NCCL INFO Trees [0] 66/32/-1->65->64 [1] 66/-1/-1->65->64 comm 0x2af228000ab0 nRanks 96 busId 26000
e08r3n04:31689:307 [2] NCCL INFO Trees [0] 67/-1/-1->66->65 [1] 67/-1/-1->66->65 comm 0x2ad454000ab0 nRanks 96 busId 43000
e08r3n04:31690:302 [3] NCCL INFO Trees [0] -1/-1/-1->67->66 [1] -1/-1/-1->67->66 comm 0x2ac5d0000ab0 nRanks 96 busId 63000
e08r3n05:26317:27342 [1] NCCL INFO Trees [0] 70/-1/-1->69->68 [1] 70/72/-1->69->68 comm 0x2b7c88000ab0 nRanks 96 busId 26000
e08r3n03:24232:25250 [2] NCCL INFO Trees [0] 63/-1/-1->62->61 [1] 63/-1/-1->62->61 comm 0x2acd78000ab0 nRanks 96 busId 43000
e08r3n05:26315:27340 [0] NCCL INFO Trees [0] 69/-1/-1->68->73 [1] 69/64/-1->68->76 comm 0x2b7410000ab0 nRanks 96 busId 4000
e08r3n07:20447:21419 [1] NCCL INFO Trees [0] 78/-1/-1->77->76 [1] 78/84/-1->77->76 comm 0x2b8d08000ab0 nRanks 96 busId 26000
e08r3n05:26318:27350 [2] NCCL INFO Trees [0] 71/-1/-1->70->69 [1] 71/-1/-1->70->69 comm 0x2b62cc000ab0 nRanks 96 busId 43000
e08r3n06:24084:25277 [2] NCCL INFO Trees [0] 75/-1/-1->74->73 [1] 75/-1/-1->74->73 comm 0x2b7908000ab0 nRanks 96 busId 43000
e08r3n07:20448:21421 [2] NCCL INFO Trees [0] 79/-1/-1->78->77 [1] 79/-1/-1->78->77 comm 0x2b4478000ab0 nRanks 96 busId 43000
e08r3n05:26319:27347 [3] NCCL INFO Trees [0] -1/-1/-1->71->70 [1] -1/-1/-1->71->70 comm 0x2ae1cc000ab0 nRanks 96 busId 63000
e08r3n06:24081:25273 [0] NCCL INFO Trees [0] 73/76/-1->72->81 [1] 73/-1/-1->72->69 comm 0x2b280c000ab0 nRanks 96 busId 4000
e08r3n07:20445:21399 [0] NCCL INFO Trees [0] 77/-1/-1->76->72 [1] 77/68/-1->76->61 comm 0x2ab264000ab0 nRanks 96 busId 4000
e08r3n06:24085:25276 [3] NCCL INFO Trees [0] -1/-1/-1->75->74 [1] -1/-1/-1->75->74 comm 0x2b6f04000ab0 nRanks 96 busId 63000
e08r3n03:24231:25246 [1] NCCL INFO Trees [0] 62/-1/-1->61->60 [1] 62/76/-1->61->60 comm 0x2b3040000ab0 nRanks 96 busId 26000
e08r3n09:10072:11133 [0] NCCL INFO Trees [0] 85/-1/-1->84->89 [1] 85/80/-1->84->77 comm 0x2ae258000ab0 nRanks 96 busId 4000
e08r3n08:14727:15954 [0] NCCL INFO Trees [0] 81/88/-1->80->64 [1] 81/-1/-1->80->84 comm 0x2ac6a8000ab0 nRanks 96 busId 4000
e08r3n10:20259:21460 [2] NCCL INFO Trees [0] 91/-1/-1->90->89 [1] 91/-1/-1->90->89 comm 0x2b837c000ab0 nRanks 96 busId 43000
e08r3n07:20449:21403 [3] NCCL INFO Trees [0] -1/-1/-1->79->78 [1] -1/-1/-1->79->78 comm 0x2b2570000ab0 nRanks 96 busId 63000
e08r3n06:24083:25278 [1] NCCL INFO Trees [0] 74/68/-1->73->72 [1] 74/-1/-1->73->72 comm 0x2acfe4000ab0 nRanks 96 busId 26000
e08r3n03:24229:25255 [0] NCCL INFO Trees [0] 61/-1/-1->60->56 [1] 61/44/-1->60->29 comm 0x2ba43c000ab0 nRanks 96 busId 4000
e08r3n09:10076:11135 [3] NCCL INFO Trees [0] -1/-1/-1->87->86 [1] -1/-1/-1->87->86 comm 0x2ab750000ab0 nRanks 96 busId 63000
e08r3n08:14731:15932 [3] NCCL INFO Trees [0] -1/-1/-1->83->82 [1] -1/-1/-1->83->82 comm 0x2b99d8000ab0 nRanks 96 busId 63000
e08r3n10:20258:21448 [1] NCCL INFO Trees [0] 90/84/-1->89->88 [1] 90/-1/-1->89->88 comm 0x2b129c000ab0 nRanks 96 busId 26000
e08r3n02:14038:15023 [3] NCCL INFO Trees [0] -1/-1/-1->59->58 [1] -1/-1/-1->59->58 comm 0x2b2d40000ab0 nRanks 96 busId 63000
e08r3n09:10074:11130 [1] NCCL INFO Trees [0] 86/-1/-1->85->84 [1] 86/88/-1->85->84 comm 0x2aed0c000ab0 nRanks 96 busId 26000
e08r3n08:14729:15948 [1] NCCL INFO Trees [0] 82/72/-1->81->80 [1] 82/-1/-1->81->80 comm 0x2b1290000ab0 nRanks 96 busId 26000
e08r3n10:20260:21455 [3] NCCL INFO Trees [0] -1/-1/-1->91->90 [1] -1/-1/-1->91->90 comm 0x2ace2c000ab0 nRanks 96 busId 63000
e08r3n02:14037:15028 [2] NCCL INFO Trees [0] 59/-1/-1->58->57 [1] 59/-1/-1->58->57 comm 0x2aecd0000ab0 nRanks 96 busId 43000
e08r3n09:10075:11136 [2] NCCL INFO Trees [0] 87/-1/-1->86->85 [1] 87/-1/-1->86->85 comm 0x2b9dc4000ab0 nRanks 96 busId 43000
e08r3n08:14730:15934 [2] NCCL INFO Trees [0] 83/-1/-1->82->81 [1] 83/-1/-1->82->81 comm 0x2aee98000ab0 nRanks 96 busId 43000
e08r3n10:20256:21447 [0] NCCL INFO Trees [0] 89/92/-1->88->80 [1] 89/-1/-1->88->85 comm 0x2b6488000ab0 nRanks 96 busId 4000
e08r3n02:14036:15025 [1] NCCL INFO Trees [0] 58/52/-1->57->56 [1] 58/-1/-1->57->56 comm 0x2b03ec000ab0 nRanks 96 busId 26000
e08r3n01:21696:22700 [3] NCCL INFO Trees [0] -1/-1/-1->55->54 [1] -1/-1/-1->55->54 comm 0x2b9c1c000ab0 nRanks 96 busId 63000
e08r3n02:14034:15017 [0] NCCL INFO Trees [0] 57/60/-1->56->48 [1] 57/-1/-1->56->53 comm 0x2b57b8000ab0 nRanks 96 busId 4000
e08r3n01:21695:22709 [2] NCCL INFO Trees [0] 55/-1/-1->54->53 [1] 55/-1/-1->54->53 comm 0x2ad544000ab0 nRanks 96 busId 43000
e08r3n11:31583:32648 [0] NCCL INFO Trees [0] 93/-1/-1->92->88 [1] 93/28/-1->92->-1 comm 0x2af294000ab0 nRanks 96 busId 4000
e08r2n08:11395:12832 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 comm 0x2b218c000ab0 nRanks 96 busId 26000
e08r3n11:31585:32654 [1] NCCL INFO Trees [0] 94/-1/-1->93->92 [1] 94/-1/-1->93->92 comm 0x2aeaa8000ab0 nRanks 96 busId 26000
e08r3n11:31587:32657 [3] NCCL INFO Trees [0] -1/-1/-1->95->94 [1] -1/-1/-1->95->94 comm 0x2b8dd8000ab0 nRanks 96 busId 63000
e08r3n04:31686:32764 [0] NCCL INFO Channel 00/0 : 63[63000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b08c0000ab0 nRanks 96
e08r3n11:31586:32658 [2] NCCL INFO Trees [0] 95/-1/-1->94->93 [1] 95/-1/-1->94->93 comm 0x2b3058000ab0 nRanks 96 busId 43000
e08r2n08:11393:12596 [0] NCCL INFO Channel 00/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
e08r2n08:11393:12596 [0] NCCL INFO Channel 01/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
e08r2n08:11393:12596 [0] NCCL INFO Trees [0] 1/64/-1->0->-1 [1] 1/-1/-1->0->4 comm 0x2afee0000ab0 nRanks 96 busId 4000
e08r2n08:11397:12837 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 comm 0x2b6a18000ab0 nRanks 96 busId 63000
e08r2n08:11396:12836 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 comm 0x2b56b4000ab0 nRanks 96 busId 43000
e08r3n01:21694:22707 [1] NCCL INFO Trees [0] 54/-1/-1->53->52 [1] 54/56/-1->53->52 comm 0x2b91cc000ab0 nRanks 96 busId 26000
e08r3n00:29328:30332 [2] NCCL INFO Trees [0] 51/-1/-1->50->49 [1] 51/-1/-1->50->49 comm 0x2ad580000ab0 nRanks 96 busId 43000
e08r3n01:21692:22698 [0] NCCL INFO Trees [0] 53/-1/-1->52->57 [1] 53/48/-1->52->45 comm 0x2accfc000ab0 nRanks 96 busId 4000
e08r3n04:31689:307 [2] NCCL INFO Channel 00 : 66[43000] -> 67[63000] via P2P/IPC comm 0x2ad454000ab0 nRanks 96
e08r3n00:29330:30333 [3] NCCL INFO Trees [0] -1/-1/-1->51->50 [1] -1/-1/-1->51->50 comm 0x2b0d6c000ab0 nRanks 96 busId 63000
e08r3n00:29327:30330 [1] NCCL INFO Trees [0] 50/40/-1->49->48 [1] 50/-1/-1->49->48 comm 0x2b149c000ab0 nRanks 96 busId 26000
e08r2n17:22822:23809 [1] NCCL INFO Trees [0] 38/-1/-1->37->36 [1] 38/40/-1->37->36 comm 0x2b7c0c000ab0 nRanks 96 busId 26000
e08r2n09:28728:29878 [0] NCCL INFO Trees [0] 5/-1/-1->4->9 [1] 5/0/-1->4->12 comm 0x2af210000ab0 nRanks 96 busId 4000
e08r2n17:22824:23810 [3] NCCL INFO Trees [0] -1/-1/-1->39->38 [1] -1/-1/-1->39->38 comm 0x2ba348000ab0 nRanks 96 busId 63000
e08r2n09:28729:29883 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/8/-1->5->4 comm 0x2b6e70000ab0 nRanks 96 busId 26000
e08r2n17:22823:23812 [2] NCCL INFO Trees [0] 39/-1/-1->38->37 [1] 39/-1/-1->38->37 comm 0x2af978000ab0 nRanks 96 busId 43000
e08r2n09:28730:29888 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 comm 0x2b25f4000ab0 nRanks 96 busId 43000
e08r2n18:2637:3714 [2] NCCL INFO Trees [0] 43/-1/-1->42->41 [1] 43/-1/-1->42->41 comm 0x2b36e4000ab0 nRanks 96 busId 43000
e08r3n04:31688:304 [1] NCCL INFO Channel 00 : 65[26000] -> 66[43000] via P2P/IPC comm 0x2af228000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Trees [0] 37/-1/-1->36->41 [1] 37/32/-1->36->44 comm 0x2b02c0000ab0 nRanks 96 busId 4000
e08r2n09:28731:29886 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 comm 0x2b67e8000ab0 nRanks 96 busId 63000
e08r2n18:2638:3717 [3] NCCL INFO Trees [0] -1/-1/-1->43->42 [1] -1/-1/-1->43->42 comm 0x2ac998000ab0 nRanks 96 busId 63000
e08r3n04:31689:307 [2] NCCL INFO Channel 01 : 66[43000] -> 67[63000] via P2P/IPC comm 0x2ad454000ab0 nRanks 96
e08r2n18:2636:3716 [1] NCCL INFO Trees [0] 42/36/-1->41->40 [1] 42/-1/-1->41->40 comm 0x2ac578000ab0 nRanks 96 busId 26000
e08r2n19:10428:11592 [1] NCCL INFO Trees [0] 46/-1/-1->45->44 [1] 46/52/-1->45->44 comm 0x2b3348000ab0 nRanks 96 busId 26000
e08r2n14:20092:21169 [0] NCCL INFO Trees [0] 25/28/-1->24->16 [1] 25/-1/-1->24->21 comm 0x2aec8c000ab0 nRanks 96 busId 4000
e08r3n00:29325:30324 [0] NCCL INFO Trees [0] 49/56/-1->48->32 [1] 49/-1/-1->48->52 comm 0x2af198000ab0 nRanks 96 busId 4000
e08r2n18:2634:3704 [0] NCCL INFO Trees [0] 41/44/-1->40->49 [1] 41/-1/-1->40->37 comm 0x2ac970000ab0 nRanks 96 busId 4000
e08r2n19:10430:11601 [3] NCCL INFO Trees [0] -1/-1/-1->47->46 [1] -1/-1/-1->47->46 comm 0x2b6930000ab0 nRanks 96 busId 63000
e08r2n14:20094:21174 [1] NCCL INFO Trees [0] 26/20/-1->25->24 [1] 26/-1/-1->25->24 comm 0x2b550c000ab0 nRanks 96 busId 26000
e08r2n19:10429:11597 [2] NCCL INFO Trees [0] 47/-1/-1->46->45 [1] 47/-1/-1->46->45 comm 0x2b019c000ab0 nRanks 96 busId 43000
e08r2n14:20095:21178 [2] NCCL INFO Trees [0] 27/-1/-1->26->25 [1] 27/-1/-1->26->25 comm 0x2bac40000ab0 nRanks 96 busId 43000
e08r2n10:2342:3554 [1] NCCL INFO Trees [0] 10/4/-1->9->8 [1] 10/-1/-1->9->8 comm 0x2ac06c000ab0 nRanks 96 busId 26000
e08r2n19:10427:11596 [0] NCCL INFO Trees [0] 45/-1/-1->44->40 [1] 45/36/-1->44->60 comm 0x2aff70000ab0 nRanks 96 busId 4000
e08r2n14:20096:21173 [3] NCCL INFO Trees [0] -1/-1/-1->27->26 [1] -1/-1/-1->27->26 comm 0x2b5e80000ab0 nRanks 96 busId 63000
e08r2n10:2344:3537 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] -1/-1/-1->11->10 comm 0x2b4a18000ab0 nRanks 96 busId 63000
e08r2n16:15463:16564 [0] NCCL INFO Trees [0] 33/48/-1->32->65 [1] 33/-1/-1->32->36 comm 0x2af018000ab0 nRanks 96 busId 4000
e08r2n10:2340:3530 [0] NCCL INFO Trees [0] 9/12/-1->8->17 [1] 9/-1/-1->8->5 comm 0x2b76f0000ab0 nRanks 96 busId 4000
e08r2n16:15465:16568 [1] NCCL INFO Trees [0] 34/16/-1->33->32 [1] 34/-1/-1->33->32 comm 0x2ad1a8000ab0 nRanks 96 busId 26000
e08r2n15:23877:25081 [0] NCCL INFO Trees [0] 29/-1/-1->28->24 [1] 29/12/-1->28->92 comm 0x2b227c000ab0 nRanks 96 busId 4000
e08r2n10:2343:3556 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 comm 0x2b14cc000ab0 nRanks 96 busId 43000
e08r2n16:15467:16571 [3] NCCL INFO Trees [0] -1/-1/-1->35->34 [1] -1/-1/-1->35->34 comm 0x2b1dc0000ab0 nRanks 96 busId 63000
e08r2n15:23879:25082 [1] NCCL INFO Trees [0] 30/-1/-1->29->28 [1] 30/60/-1->29->28 comm 0x2b0918000ab0 nRanks 96 busId 26000
e08r3n04:31688:304 [1] NCCL INFO Channel 01 : 65[26000] -> 66[43000] via P2P/IPC comm 0x2af228000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] 13/4/-1->12->28 comm 0x2b6d8c000ab0 nRanks 96 busId 4000
e08r2n16:15466:16578 [2] NCCL INFO Trees [0] 35/-1/-1->34->33 [1] 35/-1/-1->34->33 comm 0x2bafec000ab0 nRanks 96 busId 43000
e08r2n15:23881:25092 [3] NCCL INFO Trees [0] -1/-1/-1->31->30 [1] -1/-1/-1->31->30 comm 0x2b7d9c000ab0 nRanks 96 busId 63000
e08r2n11:7502:8716 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/20/-1->13->12 comm 0x2b7cdc000ab0 nRanks 96 busId 26000
e08r2n15:23880:25090 [2] NCCL INFO Trees [0] 31/-1/-1->30->29 [1] 31/-1/-1->30->29 comm 0x2b1de8000ab0 nRanks 96 busId 43000
e08r2n11:7504:8721 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] -1/-1/-1->15->14 comm 0x2baa1c000ab0 nRanks 96 busId 63000
e08r2n13:26094:27113 [1] NCCL INFO Trees [0] 22/-1/-1->21->20 [1] 22/24/-1->21->20 comm 0x2b2d08000ab0 nRanks 96 busId 26000
e08r2n11:7503:8719 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 comm 0x2b4fa4000ab0 nRanks 96 busId 43000
e08r2n13:26096:27117 [3] NCCL INFO Trees [0] -1/-1/-1->23->22 [1] -1/-1/-1->23->22 comm 0x2b83c0000ab0 nRanks 96 busId 63000
e08r2n12:10508:11619 [0] NCCL INFO Trees [0] 17/24/-1->16->33 [1] 17/-1/-1->16->20 comm 0x2b625c000ab0 nRanks 96 busId 4000
e08r2n13:26095:27111 [2] NCCL INFO Trees [0] 23/-1/-1->22->21 [1] 23/-1/-1->22->21 comm 0x2b1d90000ab0 nRanks 96 busId 43000
e08r2n12:10509:11625 [1] NCCL INFO Trees [0] 18/8/-1->17->16 [1] 18/-1/-1->17->16 comm 0x2bacb8000ab0 nRanks 96 busId 26000
e08r2n13:26093:27103 [0] NCCL INFO Trees [0] 21/-1/-1->20->25 [1] 21/16/-1->20->13 comm 0x2b19ac000ab0 nRanks 96 busId 4000
e08r2n12:10510:11624 [2] NCCL INFO Trees [0] 19/-1/-1->18->17 [1] 19/-1/-1->18->17 comm 0x2b5f60000ab0 nRanks 96 busId 43000
e08r2n12:10511:11620 [3] NCCL INFO Trees [0] -1/-1/-1->19->18 [1] -1/-1/-1->19->18 comm 0x2b3e1c000ab0 nRanks 96 busId 63000
e08r3n05:26317:27342 [1] NCCL INFO Channel 00 : 69[26000] -> 70[43000] via P2P/IPC comm 0x2b7c88000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 00/0 : 67[63000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7410000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 00/0 : 79[63000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac6a8000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 00/0 : 71[63000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b280c000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 00/0 : 75[63000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab264000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 00/0 : 59[63000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba43c000ab0 nRanks 96
e08r3n03:24231:25246 [1] NCCL INFO Channel 00 : 61[26000] -> 62[43000] via P2P/IPC comm 0x2b3040000ab0 nRanks 96
e08r3n05:26318:27350 [2] NCCL INFO Channel 00 : 70[43000] -> 71[63000] via P2P/IPC comm 0x2b62cc000ab0 nRanks 96
e08r3n10:20259:21460 [2] NCCL INFO Channel 00 : 90[43000] -> 91[63000] via P2P/IPC comm 0x2b837c000ab0 nRanks 96
e08r3n09:10074:11130 [1] NCCL INFO Channel 00 : 85[26000] -> 86[43000] via P2P/IPC comm 0x2aed0c000ab0 nRanks 96
e08r3n06:24084:25277 [2] NCCL INFO Channel 00 : 74[43000] -> 75[63000] via P2P/IPC comm 0x2b7908000ab0 nRanks 96
e08r3n10:20256:21447 [0] NCCL INFO Channel 00/0 : 87[63000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b6488000ab0 nRanks 96
e08r3n05:26317:27342 [1] NCCL INFO Channel 01 : 69[26000] -> 70[43000] via P2P/IPC comm 0x2b7c88000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 00/0 : 83[63000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae258000ab0 nRanks 96
e08r3n03:24232:25250 [2] NCCL INFO Channel 00 : 62[43000] -> 63[63000] via P2P/IPC comm 0x2acd78000ab0 nRanks 96
e08r3n07:20447:21419 [1] NCCL INFO Channel 00 : 77[26000] -> 78[43000] via P2P/IPC comm 0x2b8d08000ab0 nRanks 96
e08r3n07:20448:21421 [2] NCCL INFO Channel 00 : 78[43000] -> 79[63000] via P2P/IPC comm 0x2b4478000ab0 nRanks 96
e08r3n10:20258:21448 [1] NCCL INFO Channel 00 : 89[26000] -> 90[43000] via P2P/IPC comm 0x2b129c000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 00/0 : 55[63000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b57b8000ab0 nRanks 96
e08r3n06:24083:25278 [1] NCCL INFO Channel 00 : 73[26000] -> 74[43000] via P2P/IPC comm 0x2acfe4000ab0 nRanks 96
e08r3n10:20259:21460 [2] NCCL INFO Channel 01 : 90[43000] -> 91[63000] via P2P/IPC comm 0x2b837c000ab0 nRanks 96
e08r3n02:14037:15028 [2] NCCL INFO Channel 00 : 58[43000] -> 59[63000] via P2P/IPC comm 0x2aecd0000ab0 nRanks 96
e08r3n08:14730:15934 [2] NCCL INFO Channel 00 : 82[43000] -> 83[63000] via P2P/IPC comm 0x2aee98000ab0 nRanks 96
e08r3n05:26318:27350 [2] NCCL INFO Channel 01 : 70[43000] -> 71[63000] via P2P/IPC comm 0x2b62cc000ab0 nRanks 96
e08r3n09:10074:11130 [1] NCCL INFO Channel 01 : 85[26000] -> 86[43000] via P2P/IPC comm 0x2aed0c000ab0 nRanks 96
e08r3n08:14729:15948 [1] NCCL INFO Channel 00 : 81[26000] -> 82[43000] via P2P/IPC comm 0x2b1290000ab0 nRanks 96
e08r3n11:31583:32648 [0] NCCL INFO Channel 00/0 : 91[63000] -> 92[4000] [receive] via NET/IB/0 comm 0x2af294000ab0 nRanks 96
e08r3n06:24084:25277 [2] NCCL INFO Channel 01 : 74[43000] -> 75[63000] via P2P/IPC comm 0x2b7908000ab0 nRanks 96
e08r3n07:20447:21419 [1] NCCL INFO Channel 01 : 77[26000] -> 78[43000] via P2P/IPC comm 0x2b8d08000ab0 nRanks 96
e08r3n09:10075:11136 [2] NCCL INFO Channel 00 : 86[43000] -> 87[63000] via P2P/IPC comm 0x2b9dc4000ab0 nRanks 96
e08r3n10:20258:21448 [1] NCCL INFO Channel 01 : 89[26000] -> 90[43000] via P2P/IPC comm 0x2b129c000ab0 nRanks 96
e08r3n03:24231:25246 [1] NCCL INFO Channel 01 : 61[26000] -> 62[43000] via P2P/IPC comm 0x2b3040000ab0 nRanks 96
e08r3n02:14036:15025 [1] NCCL INFO Channel 00 : 57[26000] -> 58[43000] via P2P/IPC comm 0x2b03ec000ab0 nRanks 96
e08r3n01:21695:22709 [2] NCCL INFO Channel 00 : 54[43000] -> 55[63000] via P2P/IPC comm 0x2ad544000ab0 nRanks 96
e08r3n03:24232:25250 [2] NCCL INFO Channel 01 : 62[43000] -> 63[63000] via P2P/IPC comm 0x2acd78000ab0 nRanks 96
e08r3n02:14037:15028 [2] NCCL INFO Channel 01 : 58[43000] -> 59[63000] via P2P/IPC comm 0x2aecd0000ab0 nRanks 96
e08r3n08:14729:15948 [1] NCCL INFO Channel 01 : 81[26000] -> 82[43000] via P2P/IPC comm 0x2b1290000ab0 nRanks 96
e08r3n11:31585:32654 [1] NCCL INFO Channel 00 : 93[26000] -> 94[43000] via P2P/IPC comm 0x2aeaa8000ab0 nRanks 96
e08r3n06:24083:25278 [1] NCCL INFO Channel 01 : 73[26000] -> 74[43000] via P2P/IPC comm 0x2acfe4000ab0 nRanks 96
e08r3n07:20448:21421 [2] NCCL INFO Channel 01 : 78[43000] -> 79[63000] via P2P/IPC comm 0x2b4478000ab0 nRanks 96
e08r2n08:11395:12832 [1] NCCL INFO Channel 00 : 1[26000] -> 2[43000] via P2P/IPC comm 0x2b218c000ab0 nRanks 96
e08r2n08:11393:12596 [0] NCCL INFO Channel 00/0 : 95[63000] -> 0[4000] [receive] via NET/IB/0 comm 0x2afee0000ab0 nRanks 96
e08r3n08:14730:15934 [2] NCCL INFO Channel 01 : 82[43000] -> 83[63000] via P2P/IPC comm 0x2aee98000ab0 nRanks 96
e08r3n11:31586:32658 [2] NCCL INFO Channel 00 : 94[43000] -> 95[63000] via P2P/IPC comm 0x2b3058000ab0 nRanks 96
e08r3n01:21695:22709 [2] NCCL INFO Channel 01 : 54[43000] -> 55[63000] via P2P/IPC comm 0x2ad544000ab0 nRanks 96
e08r3n11:31585:32654 [1] NCCL INFO Channel 01 : 93[26000] -> 94[43000] via P2P/IPC comm 0x2aeaa8000ab0 nRanks 96
e08r3n01:21694:22707 [1] NCCL INFO Channel 00 : 53[26000] -> 54[43000] via P2P/IPC comm 0x2b91cc000ab0 nRanks 96
e08r3n09:10075:11136 [2] NCCL INFO Channel 01 : 86[43000] -> 87[63000] via P2P/IPC comm 0x2b9dc4000ab0 nRanks 96
e08r3n02:14036:15025 [1] NCCL INFO Channel 01 : 57[26000] -> 58[43000] via P2P/IPC comm 0x2b03ec000ab0 nRanks 96
e08r2n08:11395:12832 [1] NCCL INFO Channel 01 : 1[26000] -> 2[43000] via P2P/IPC comm 0x2b218c000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 00/0 : 51[63000] -> 52[4000] [receive] via NET/IB/0 comm 0x2accfc000ab0 nRanks 96
e08r3n00:29327:30330 [1] NCCL INFO Channel 00 : 49[26000] -> 50[43000] via P2P/IPC comm 0x2b149c000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 00/0 : 23[63000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aec8c000ab0 nRanks 96
e08r2n08:11396:12836 [2] NCCL INFO Channel 00 : 2[43000] -> 3[63000] via P2P/IPC comm 0x2b56b4000ab0 nRanks 96
e08r3n01:21694:22707 [1] NCCL INFO Channel 01 : 53[26000] -> 54[43000] via P2P/IPC comm 0x2b91cc000ab0 nRanks 96
e08r3n00:29328:30332 [2] NCCL INFO Channel 00 : 50[43000] -> 51[63000] via P2P/IPC comm 0x2ad580000ab0 nRanks 96
e08r2n09:28729:29883 [1] NCCL INFO Channel 00 : 5[26000] -> 6[43000] via P2P/IPC comm 0x2b6e70000ab0 nRanks 96
e08r3n11:31586:32658 [2] NCCL INFO Channel 01 : 94[43000] -> 95[63000] via P2P/IPC comm 0x2b3058000ab0 nRanks 96
e08r2n17:22822:23809 [1] NCCL INFO Channel 00 : 37[26000] -> 38[43000] via P2P/IPC comm 0x2b7c0c000ab0 nRanks 96
e08r2n18:2636:3716 [1] NCCL INFO Channel 00 : 41[26000] -> 42[43000] via P2P/IPC comm 0x2ac578000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 00/0 : 47[63000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af198000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 00/0 : 31[63000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af018000ab0 nRanks 96
e08r2n19:10428:11592 [1] NCCL INFO Channel 00 : 45[26000] -> 46[43000] via P2P/IPC comm 0x2b3348000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 00/0 : 43[63000] -> 44[4000] [receive] via NET/IB/0 comm 0x2aff70000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 00/0 : 3[63000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af210000ab0 nRanks 96
e08r2n18:2637:3714 [2] NCCL INFO Channel 00 : 42[43000] -> 43[63000] via P2P/IPC comm 0x2b36e4000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Channel 00/0 : 35[63000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b02c0000ab0 nRanks 96
e08r2n08:11396:12836 [2] NCCL INFO Channel 01 : 2[43000] -> 3[63000] via P2P/IPC comm 0x2b56b4000ab0 nRanks 96
e08r2n09:28730:29888 [2] NCCL INFO Channel 00 : 6[43000] -> 7[63000] via P2P/IPC comm 0x2b25f4000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 00/0 : 39[63000] -> 40[4000] [receive] via NET/IB/0 comm 0x2ac970000ab0 nRanks 96
e08r2n17:22823:23812 [2] NCCL INFO Channel 00 : 38[43000] -> 39[63000] via P2P/IPC comm 0x2af978000ab0 nRanks 96
e08r3n00:29328:30332 [2] NCCL INFO Channel 01 : 50[43000] -> 51[63000] via P2P/IPC comm 0x2ad580000ab0 nRanks 96
e08r2n10:2342:3554 [1] NCCL INFO Channel 00 : 9[26000] -> 10[43000] via P2P/IPC comm 0x2ac06c000ab0 nRanks 96
e08r2n09:28729:29883 [1] NCCL INFO Channel 01 : 5[26000] -> 6[43000] via P2P/IPC comm 0x2b6e70000ab0 nRanks 96
e08r2n18:2636:3716 [1] NCCL INFO Channel 01 : 41[26000] -> 42[43000] via P2P/IPC comm 0x2ac578000ab0 nRanks 96
e08r2n17:22822:23809 [1] NCCL INFO Channel 01 : 37[26000] -> 38[43000] via P2P/IPC comm 0x2b7c0c000ab0 nRanks 96
e08r3n00:29327:30330 [1] NCCL INFO Channel 01 : 49[26000] -> 50[43000] via P2P/IPC comm 0x2b149c000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 00/0 : 7[63000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b76f0000ab0 nRanks 96
e08r2n14:20094:21174 [1] NCCL INFO Channel 00 : 25[26000] -> 26[43000] via P2P/IPC comm 0x2b550c000ab0 nRanks 96
e08r2n16:15465:16568 [1] NCCL INFO Channel 00 : 33[26000] -> 34[43000] via P2P/IPC comm 0x2ad1a8000ab0 nRanks 96
e08r2n19:10428:11592 [1] NCCL INFO Channel 01 : 45[26000] -> 46[43000] via P2P/IPC comm 0x2b3348000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 00/0 : 11[63000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6d8c000ab0 nRanks 96
e08r2n15:23879:25082 [1] NCCL INFO Channel 00 : 29[26000] -> 30[43000] via P2P/IPC comm 0x2b0918000ab0 nRanks 96
e08r2n09:28730:29888 [2] NCCL INFO Channel 01 : 6[43000] -> 7[63000] via P2P/IPC comm 0x2b25f4000ab0 nRanks 96
e08r2n18:2637:3714 [2] NCCL INFO Channel 01 : 42[43000] -> 43[63000] via P2P/IPC comm 0x2b36e4000ab0 nRanks 96
e08r2n17:22823:23812 [2] NCCL INFO Channel 01 : 38[43000] -> 39[63000] via P2P/IPC comm 0x2af978000ab0 nRanks 96
e08r2n10:2343:3556 [2] NCCL INFO Channel 00 : 10[43000] -> 11[63000] via P2P/IPC comm 0x2b14cc000ab0 nRanks 96
e08r2n14:20095:21178 [2] NCCL INFO Channel 00 : 26[43000] -> 27[63000] via P2P/IPC comm 0x2bac40000ab0 nRanks 96
e08r2n16:15466:16578 [2] NCCL INFO Channel 00 : 34[43000] -> 35[63000] via P2P/IPC comm 0x2bafec000ab0 nRanks 96
e08r2n19:10429:11597 [2] NCCL INFO Channel 00 : 46[43000] -> 47[63000] via P2P/IPC comm 0x2b019c000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 00/0 : 27[63000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b227c000ab0 nRanks 96
e08r2n10:2342:3554 [1] NCCL INFO Channel 01 : 9[26000] -> 10[43000] via P2P/IPC comm 0x2ac06c000ab0 nRanks 96
e08r2n14:20094:21174 [1] NCCL INFO Channel 01 : 25[26000] -> 26[43000] via P2P/IPC comm 0x2b550c000ab0 nRanks 96
e08r2n16:15466:16578 [2] NCCL INFO Channel 01 : 34[43000] -> 35[63000] via P2P/IPC comm 0x2bafec000ab0 nRanks 96
e08r2n19:10429:11597 [2] NCCL INFO Channel 01 : 46[43000] -> 47[63000] via P2P/IPC comm 0x2b019c000ab0 nRanks 96
e08r2n15:23879:25082 [1] NCCL INFO Channel 01 : 29[26000] -> 30[43000] via P2P/IPC comm 0x2b0918000ab0 nRanks 96
e08r2n11:7503:8719 [2] NCCL INFO Channel 00 : 14[43000] -> 15[63000] via P2P/IPC comm 0x2b4fa4000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 00/0 : 15[63000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b625c000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 00/0 : 19[63000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b19ac000ab0 nRanks 96
e08r2n10:2343:3556 [2] NCCL INFO Channel 01 : 10[43000] -> 11[63000] via P2P/IPC comm 0x2b14cc000ab0 nRanks 96
e08r2n14:20095:21178 [2] NCCL INFO Channel 01 : 26[43000] -> 27[63000] via P2P/IPC comm 0x2bac40000ab0 nRanks 96
e08r2n16:15465:16568 [1] NCCL INFO Channel 01 : 33[26000] -> 34[43000] via P2P/IPC comm 0x2ad1a8000ab0 nRanks 96
e08r2n15:23880:25090 [2] NCCL INFO Channel 00 : 30[43000] -> 31[63000] via P2P/IPC comm 0x2b1de8000ab0 nRanks 96
e08r2n11:7502:8716 [1] NCCL INFO Channel 00 : 13[26000] -> 14[43000] via P2P/IPC comm 0x2b7cdc000ab0 nRanks 96
e08r2n13:26095:27111 [2] NCCL INFO Channel 00 : 22[43000] -> 23[63000] via P2P/IPC comm 0x2b1d90000ab0 nRanks 96
e08r2n15:23880:25090 [2] NCCL INFO Channel 01 : 30[43000] -> 31[63000] via P2P/IPC comm 0x2b1de8000ab0 nRanks 96
e08r2n11:7503:8719 [2] NCCL INFO Channel 01 : 14[43000] -> 15[63000] via P2P/IPC comm 0x2b4fa4000ab0 nRanks 96
e08r2n13:26094:27113 [1] NCCL INFO Channel 00 : 21[26000] -> 22[43000] via P2P/IPC comm 0x2b2d08000ab0 nRanks 96
e08r2n12:10509:11625 [1] NCCL INFO Channel 00 : 17[26000] -> 18[43000] via P2P/IPC comm 0x2bacb8000ab0 nRanks 96
e08r2n11:7502:8716 [1] NCCL INFO Channel 01 : 13[26000] -> 14[43000] via P2P/IPC comm 0x2b7cdc000ab0 nRanks 96
e08r2n13:26095:27111 [2] NCCL INFO Channel 01 : 22[43000] -> 23[63000] via P2P/IPC comm 0x2b1d90000ab0 nRanks 96
e08r2n12:10510:11624 [2] NCCL INFO Channel 00 : 18[43000] -> 19[63000] via P2P/IPC comm 0x2b5f60000ab0 nRanks 96
e08r2n12:10509:11625 [1] NCCL INFO Channel 01 : 17[26000] -> 18[43000] via P2P/IPC comm 0x2bacb8000ab0 nRanks 96
e08r2n13:26094:27113 [1] NCCL INFO Channel 01 : 21[26000] -> 22[43000] via P2P/IPC comm 0x2b2d08000ab0 nRanks 96
e08r2n12:10510:11624 [2] NCCL INFO Channel 01 : 18[43000] -> 19[63000] via P2P/IPC comm 0x2b5f60000ab0 nRanks 96
e08r3n04:31690:302 [3] NCCL INFO Channel 00/0 : 67[63000] -> 68[4000] [send] via NET/IB/0 comm 0x2ac5d0000ab0 nRanks 96
e08r3n04:31686:32764 [0] NCCL INFO Channel 01/0 : 63[63000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b08c0000ab0 nRanks 96
e08r3n03:24233:25253 [3] NCCL INFO Channel 00/0 : 63[63000] -> 64[4000] [send] via NET/IB/0 comm 0x2b66e0000ab0 nRanks 96
e08r3n04:31686:32764 [0] NCCL INFO Channel 00 : 64[4000] -> 65[26000] via P2P/IPC comm 0x2b08c0000ab0 nRanks 96
e08r3n04:31686:32764 [0] NCCL INFO Channel 01 : 64[4000] -> 65[26000] via P2P/IPC comm 0x2b08c0000ab0 nRanks 96
e08r3n05:26319:27347 [3] NCCL INFO Channel 00/0 : 71[63000] -> 72[4000] [send] via NET/IB/0 comm 0x2ae1cc000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 01/0 : 79[63000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac6a8000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 00 : 80[4000] -> 81[26000] via P2P/IPC comm 0x2ac6a8000ab0 nRanks 96
e08r3n09:10076:11135 [3] NCCL INFO Channel 00/0 : 87[63000] -> 88[4000] [send] via NET/IB/0 comm 0x2ab750000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 01/0 : 67[63000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7410000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 01/0 : 83[63000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae258000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 00 : 68[4000] -> 69[26000] via P2P/IPC comm 0x2b7410000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 00 : 84[4000] -> 85[26000] via P2P/IPC comm 0x2ae258000ab0 nRanks 96
e08r3n07:20449:21403 [3] NCCL INFO Channel 00/0 : 79[63000] -> 80[4000] [send] via NET/IB/0 comm 0x2b2570000ab0 nRanks 96
e08r3n06:24085:25276 [3] NCCL INFO Channel 00/0 : 75[63000] -> 76[4000] [send] via NET/IB/0 comm 0x2b6f04000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 01/0 : 75[63000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab264000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 01/0 : 71[63000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b280c000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 00 : 72[4000] -> 73[26000] via P2P/IPC comm 0x2b280c000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 00 : 76[4000] -> 77[26000] via P2P/IPC comm 0x2ab264000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 01 : 80[4000] -> 81[26000] via P2P/IPC comm 0x2ac6a8000ab0 nRanks 96
e08r3n01:21696:22700 [3] NCCL INFO Channel 00/0 : 55[63000] -> 56[4000] [send] via NET/IB/0 comm 0x2b9c1c000ab0 nRanks 96
e08r3n10:20260:21455 [3] NCCL INFO Channel 00/0 : 91[63000] -> 92[4000] [send] via NET/IB/0 comm 0x2ace2c000ab0 nRanks 96
e08r3n10:20256:21447 [0] NCCL INFO Channel 01/0 : 87[63000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b6488000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 01 : 68[4000] -> 69[26000] via P2P/IPC comm 0x2b7410000ab0 nRanks 96
e08r3n10:20256:21447 [0] NCCL INFO Channel 00 : 88[4000] -> 89[26000] via P2P/IPC comm 0x2b6488000ab0 nRanks 96
e08r3n02:14038:15023 [3] NCCL INFO Channel 00/0 : 59[63000] -> 60[4000] [send] via NET/IB/0 comm 0x2b2d40000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 01/0 : 55[63000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b57b8000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 01 : 84[4000] -> 85[26000] via P2P/IPC comm 0x2ae258000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 00 : 56[4000] -> 57[26000] via P2P/IPC comm 0x2b57b8000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 01 : 72[4000] -> 73[26000] via P2P/IPC comm 0x2b280c000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 01/0 : 59[63000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba43c000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 00 : 60[4000] -> 61[26000] via P2P/IPC comm 0x2ba43c000ab0 nRanks 96
e08r3n08:14731:15932 [3] NCCL INFO Channel 00/0 : 83[63000] -> 84[4000] [send] via NET/IB/0 comm 0x2b99d8000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 01 : 76[4000] -> 77[26000] via P2P/IPC comm 0x2ab264000ab0 nRanks 96
e08r3n10:20256:21447 [0] NCCL INFO Channel 01 : 88[4000] -> 89[26000] via P2P/IPC comm 0x2b6488000ab0 nRanks 96
e08r2n08:11397:12837 [3] NCCL INFO Channel 00/0 : 3[63000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6a18000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 01/0 : 23[63000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aec8c000ab0 nRanks 96
e08r2n17:22824:23810 [3] NCCL INFO Channel 00/0 : 39[63000] -> 40[4000] [send] via NET/IB/0 comm 0x2ba348000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 00 : 24[4000] -> 25[26000] via P2P/IPC comm 0x2aec8c000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 01 : 56[4000] -> 57[26000] via P2P/IPC comm 0x2b57b8000ab0 nRanks 96
e08r2n08:11393:12596 [0] NCCL INFO Channel 01/0 : 95[63000] -> 0[4000] [receive] via NET/IB/0 comm 0x2afee0000ab0 nRanks 96
e08r2n08:11393:12596 [0] NCCL INFO Channel 00 : 0[4000] -> 1[26000] via P2P/IPC comm 0x2afee0000ab0 nRanks 96
e08r3n11:31587:32657 [3] NCCL INFO Channel 00/0 : 95[63000] -> 0[4000] [send] via NET/IB/0 comm 0x2b8dd8000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 01 : 60[4000] -> 61[26000] via P2P/IPC comm 0x2ba43c000ab0 nRanks 96
e08r3n11:31583:32648 [0] NCCL INFO Channel 01/0 : 91[63000] -> 92[4000] [receive] via NET/IB/0 comm 0x2af294000ab0 nRanks 96
e08r3n11:31583:32648 [0] NCCL INFO Channel 00 : 92[4000] -> 93[26000] via P2P/IPC comm 0x2af294000ab0 nRanks 96
e08r2n19:10430:11601 [3] NCCL INFO Channel 00/0 : 47[63000] -> 48[4000] [send] via NET/IB/0 comm 0x2b6930000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 01 : 24[4000] -> 25[26000] via P2P/IPC comm 0x2aec8c000ab0 nRanks 96
e08r2n18:2638:3717 [3] NCCL INFO Channel 00/0 : 43[63000] -> 44[4000] [send] via NET/IB/0 comm 0x2ac998000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 01/0 : 51[63000] -> 52[4000] [receive] via NET/IB/0 comm 0x2accfc000ab0 nRanks 96
e08r2n09:28731:29886 [3] NCCL INFO Channel 00/0 : 7[63000] -> 8[4000] [send] via NET/IB/0 comm 0x2b67e8000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 00 : 52[4000] -> 53[26000] via P2P/IPC comm 0x2accfc000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 01/0 : 43[63000] -> 44[4000] [receive] via NET/IB/0 comm 0x2aff70000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 01/0 : 39[63000] -> 40[4000] [receive] via NET/IB/0 comm 0x2ac970000ab0 nRanks 96
e08r2n08:11393:12596 [0] NCCL INFO Channel 01 : 0[4000] -> 1[26000] via P2P/IPC comm 0x2afee0000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 00 : 44[4000] -> 45[26000] via P2P/IPC comm 0x2aff70000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 00 : 40[4000] -> 41[26000] via P2P/IPC comm 0x2ac970000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 01/0 : 3[63000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af210000ab0 nRanks 96
e08r3n00:29330:30333 [3] NCCL INFO Channel 00/0 : 51[63000] -> 52[4000] [send] via NET/IB/0 comm 0x2b0d6c000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 00 : 4[4000] -> 5[26000] via P2P/IPC comm 0x2af210000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 01/0 : 47[63000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af198000ab0 nRanks 96
e08r2n16:15467:16571 [3] NCCL INFO Channel 00/0 : 35[63000] -> 36[4000] [send] via NET/IB/0 comm 0x2b1dc0000ab0 nRanks 96
e08r3n11:31583:32648 [0] NCCL INFO Channel 01 : 92[4000] -> 93[26000] via P2P/IPC comm 0x2af294000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 00 : 48[4000] -> 49[26000] via P2P/IPC comm 0x2af198000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 01/0 : 31[63000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af018000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 00 : 32[4000] -> 33[26000] via P2P/IPC comm 0x2af018000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 01/0 : 15[63000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b625c000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 01 : 44[4000] -> 45[26000] via P2P/IPC comm 0x2aff70000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 00 : 16[4000] -> 17[26000] via P2P/IPC comm 0x2b625c000ab0 nRanks 96
e08r2n10:2344:3537 [3] NCCL INFO Channel 00/0 : 11[63000] -> 12[4000] [send] via NET/IB/0 comm 0x2b4a18000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 01/0 : 7[63000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b76f0000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 01 : 52[4000] -> 53[26000] via P2P/IPC comm 0x2accfc000ab0 nRanks 96
e08r3n04:31688:304 [1] NCCL INFO Connected all rings comm 0x2af228000ab0 nRanks 96 busId 26000
e08r2n13:26096:27117 [3] NCCL INFO Channel 00/0 : 23[63000] -> 24[4000] [send] via NET/IB/0 comm 0x2b83c0000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 00 : 8[4000] -> 9[26000] via P2P/IPC comm 0x2b76f0000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 01/0 : 19[63000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b19ac000ab0 nRanks 96
e08r2n14:20096:21173 [3] NCCL INFO Channel 00/0 : 27[63000] -> 28[4000] [send] via NET/IB/0 comm 0x2b5e80000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 01 : 4[4000] -> 5[26000] via P2P/IPC comm 0x2af210000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 00 : 20[4000] -> 21[26000] via P2P/IPC comm 0x2b19ac000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 01 : 48[4000] -> 49[26000] via P2P/IPC comm 0x2af198000ab0 nRanks 96
e08r2n15:23881:25092 [3] NCCL INFO Channel 00/0 : 31[63000] -> 32[4000] [send] via NET/IB/0 comm 0x2b7d9c000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 01 : 40[4000] -> 41[26000] via P2P/IPC comm 0x2ac970000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 01 : 32[4000] -> 33[26000] via P2P/IPC comm 0x2af018000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Channel 01/0 : 35[63000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b02c0000ab0 nRanks 96
e08r2n11:7504:8721 [3] NCCL INFO Channel 00/0 : 15[63000] -> 16[4000] [send] via NET/IB/0 comm 0x2baa1c000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Channel 00 : 36[4000] -> 37[26000] via P2P/IPC comm 0x2b02c0000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 01/0 : 11[63000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6d8c000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 00 : 12[4000] -> 13[26000] via P2P/IPC comm 0x2b6d8c000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 01 : 16[4000] -> 17[26000] via P2P/IPC comm 0x2b625c000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 01/0 : 27[63000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b227c000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 00 : 28[4000] -> 29[26000] via P2P/IPC comm 0x2b227c000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 01 : 8[4000] -> 9[26000] via P2P/IPC comm 0x2b76f0000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 01 : 12[4000] -> 13[26000] via P2P/IPC comm 0x2b6d8c000ab0 nRanks 96
e08r3n04:31688:304 [1] NCCL INFO Channel 00/0 : 32[4000] -> 65[26000] [receive] via NET/IB/0 comm 0x2af228000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 01 : 20[4000] -> 21[26000] via P2P/IPC comm 0x2b19ac000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 01 : 28[4000] -> 29[26000] via P2P/IPC comm 0x2b227c000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Channel 01 : 36[4000] -> 37[26000] via P2P/IPC comm 0x2b02c0000ab0 nRanks 96
e08r2n12:10511:11620 [3] NCCL INFO Channel 00/0 : 19[63000] -> 20[4000] [send] via NET/IB/0 comm 0x2b3e1c000ab0 nRanks 96
e08r3n05:26317:27342 [1] NCCL INFO Connected all rings comm 0x2b7c88000ab0 nRanks 96 busId 26000
e08r3n09:10074:11130 [1] NCCL INFO Connected all rings comm 0x2aed0c000ab0 nRanks 96 busId 26000
e08r3n08:14729:15948 [1] NCCL INFO Connected all rings comm 0x2b1290000ab0 nRanks 96 busId 26000
e08r3n07:20447:21419 [1] NCCL INFO Connected all rings comm 0x2b8d08000ab0 nRanks 96 busId 26000
e08r3n10:20258:21448 [1] NCCL INFO Connected all rings comm 0x2b129c000ab0 nRanks 96 busId 26000
e08r3n05:26317:27342 [1] NCCL INFO Channel 01/0 : 69[26000] -> 72[4000] [send] via NET/IB/0 comm 0x2b7c88000ab0 nRanks 96
e08r3n06:24083:25278 [1] NCCL INFO Connected all rings comm 0x2acfe4000ab0 nRanks 96 busId 26000
e08r3n09:10074:11130 [1] NCCL INFO Channel 01/0 : 85[26000] -> 88[4000] [send] via NET/IB/0 comm 0x2aed0c000ab0 nRanks 96
e08r3n07:20447:21419 [1] NCCL INFO Channel 01/0 : 77[26000] -> 84[4000] [send] via NET/IB/0 comm 0x2b8d08000ab0 nRanks 96
e08r2n14:20094:21174 [1] NCCL INFO Connected all rings comm 0x2b550c000ab0 nRanks 96 busId 26000
e08r3n08:14729:15948 [1] NCCL INFO Channel 00/0 : 72[4000] -> 81[26000] [receive] via NET/IB/0 comm 0x2b1290000ab0 nRanks 96
e08r3n10:20258:21448 [1] NCCL INFO Channel 00/0 : 84[4000] -> 89[26000] [receive] via NET/IB/0 comm 0x2b129c000ab0 nRanks 96
e08r3n11:31585:32654 [1] NCCL INFO Connected all rings comm 0x2aeaa8000ab0 nRanks 96 busId 26000
e08r3n02:14036:15025 [1] NCCL INFO Connected all rings comm 0x2b03ec000ab0 nRanks 96 busId 26000
e08r3n03:24231:25246 [1] NCCL INFO Connected all rings comm 0x2b3040000ab0 nRanks 96 busId 26000
e08r3n06:24083:25278 [1] NCCL INFO Channel 00/0 : 68[4000] -> 73[26000] [receive] via NET/IB/0 comm 0x2acfe4000ab0 nRanks 96
e08r3n01:21694:22707 [1] NCCL INFO Connected all rings comm 0x2b91cc000ab0 nRanks 96 busId 26000
e08r2n08:11395:12832 [1] NCCL INFO Connected all rings comm 0x2b218c000ab0 nRanks 96 busId 26000
e08r2n14:20094:21174 [1] NCCL INFO Channel 00/0 : 20[4000] -> 25[26000] [receive] via NET/IB/0 comm 0x2b550c000ab0 nRanks 96
e08r3n03:24231:25246 [1] NCCL INFO Channel 01/0 : 61[26000] -> 76[4000] [send] via NET/IB/0 comm 0x2b3040000ab0 nRanks 96
e08r2n19:10428:11592 [1] NCCL INFO Connected all rings comm 0x2b3348000ab0 nRanks 96 busId 26000
e08r3n02:14036:15025 [1] NCCL INFO Channel 00/0 : 52[4000] -> 57[26000] [receive] via NET/IB/0 comm 0x2b03ec000ab0 nRanks 96
e08r3n01:21694:22707 [1] NCCL INFO Channel 01/0 : 53[26000] -> 56[4000] [send] via NET/IB/0 comm 0x2b91cc000ab0 nRanks 96
e08r2n09:28729:29883 [1] NCCL INFO Connected all rings comm 0x2b6e70000ab0 nRanks 96 busId 26000
e08r3n11:31585:32654 [1] NCCL INFO Channel 00 : 93[26000] -> 92[4000] via P2P/IPC comm 0x2aeaa8000ab0 nRanks 96
e08r3n00:29327:30330 [1] NCCL INFO Connected all rings comm 0x2b149c000ab0 nRanks 96 busId 26000
e08r2n19:10428:11592 [1] NCCL INFO Channel 01/0 : 45[26000] -> 52[4000] [send] via NET/IB/0 comm 0x2b3348000ab0 nRanks 96
e08r2n16:15465:16568 [1] NCCL INFO Connected all rings comm 0x2ad1a8000ab0 nRanks 96 busId 26000
e08r3n11:31585:32654 [1] NCCL INFO Channel 01 : 93[26000] -> 92[4000] via P2P/IPC comm 0x2aeaa8000ab0 nRanks 96
e08r2n08:11395:12832 [1] NCCL INFO Channel 00 : 1[26000] -> 0[4000] via P2P/IPC comm 0x2b218c000ab0 nRanks 96
e08r2n12:10509:11625 [1] NCCL INFO Connected all rings comm 0x2bacb8000ab0 nRanks 96 busId 26000
e08r2n11:7502:8716 [1] NCCL INFO Connected all rings comm 0x2b7cdc000ab0 nRanks 96 busId 26000
e08r2n18:2636:3716 [1] NCCL INFO Connected all rings comm 0x2ac578000ab0 nRanks 96 busId 26000
e08r2n15:23879:25082 [1] NCCL INFO Connected all rings comm 0x2b0918000ab0 nRanks 96 busId 26000
e08r2n09:28729:29883 [1] NCCL INFO Channel 01/0 : 5[26000] -> 8[4000] [send] via NET/IB/0 comm 0x2b6e70000ab0 nRanks 96
e08r2n08:11395:12832 [1] NCCL INFO Channel 01 : 1[26000] -> 0[4000] via P2P/IPC comm 0x2b218c000ab0 nRanks 96
e08r2n10:2342:3554 [1] NCCL INFO Connected all rings comm 0x2ac06c000ab0 nRanks 96 busId 26000
e08r2n13:26094:27113 [1] NCCL INFO Connected all rings comm 0x2b2d08000ab0 nRanks 96 busId 26000
e08r3n00:29327:30330 [1] NCCL INFO Channel 00/0 : 40[4000] -> 49[26000] [receive] via NET/IB/0 comm 0x2b149c000ab0 nRanks 96
e08r2n16:15465:16568 [1] NCCL INFO Channel 00/0 : 16[4000] -> 33[26000] [receive] via NET/IB/0 comm 0x2ad1a8000ab0 nRanks 96
e08r2n11:7502:8716 [1] NCCL INFO Channel 01/0 : 13[26000] -> 20[4000] [send] via NET/IB/0 comm 0x2b7cdc000ab0 nRanks 96
e08r2n17:22822:23809 [1] NCCL INFO Connected all rings comm 0x2b7c0c000ab0 nRanks 96 busId 26000
e08r2n15:23879:25082 [1] NCCL INFO Channel 01/0 : 29[26000] -> 60[4000] [send] via NET/IB/0 comm 0x2b0918000ab0 nRanks 96
e08r2n12:10509:11625 [1] NCCL INFO Channel 00/0 : 8[4000] -> 17[26000] [receive] via NET/IB/0 comm 0x2bacb8000ab0 nRanks 96
e08r2n18:2636:3716 [1] NCCL INFO Channel 00/0 : 36[4000] -> 41[26000] [receive] via NET/IB/0 comm 0x2ac578000ab0 nRanks 96
e08r2n13:26094:27113 [1] NCCL INFO Channel 01/0 : 21[26000] -> 24[4000] [send] via NET/IB/0 comm 0x2b2d08000ab0 nRanks 96
e08r2n10:2342:3554 [1] NCCL INFO Channel 00/0 : 4[4000] -> 9[26000] [receive] via NET/IB/0 comm 0x2ac06c000ab0 nRanks 96
e08r2n17:22822:23809 [1] NCCL INFO Channel 01/0 : 37[26000] -> 40[4000] [send] via NET/IB/0 comm 0x2b7c0c000ab0 nRanks 96
e08r3n04:31690:302 [3] NCCL INFO Channel 01/0 : 67[63000] -> 68[4000] [send] via NET/IB/0 comm 0x2ac5d0000ab0 nRanks 96
e08r3n03:24233:25253 [3] NCCL INFO Channel 01/0 : 63[63000] -> 64[4000] [send] via NET/IB/0 comm 0x2b66e0000ab0 nRanks 96
e08r3n05:26319:27347 [3] NCCL INFO Channel 01/0 : 71[63000] -> 72[4000] [send] via NET/IB/0 comm 0x2ae1cc000ab0 nRanks 96
e08r3n09:10076:11135 [3] NCCL INFO Channel 01/0 : 87[63000] -> 88[4000] [send] via NET/IB/0 comm 0x2ab750000ab0 nRanks 96
e08r3n07:20449:21403 [3] NCCL INFO Channel 01/0 : 79[63000] -> 80[4000] [send] via NET/IB/0 comm 0x2b2570000ab0 nRanks 96
e08r3n06:24085:25276 [3] NCCL INFO Channel 01/0 : 75[63000] -> 76[4000] [send] via NET/IB/0 comm 0x2b6f04000ab0 nRanks 96
e08r3n10:20260:21455 [3] NCCL INFO Channel 01/0 : 91[63000] -> 92[4000] [send] via NET/IB/0 comm 0x2ace2c000ab0 nRanks 96
e08r3n01:21696:22700 [3] NCCL INFO Channel 01/0 : 55[63000] -> 56[4000] [send] via NET/IB/0 comm 0x2b9c1c000ab0 nRanks 96
e08r3n02:14038:15023 [3] NCCL INFO Channel 01/0 : 59[63000] -> 60[4000] [send] via NET/IB/0 comm 0x2b2d40000ab0 nRanks 96
e08r3n08:14731:15932 [3] NCCL INFO Channel 01/0 : 83[63000] -> 84[4000] [send] via NET/IB/0 comm 0x2b99d8000ab0 nRanks 96
e08r2n08:11397:12837 [3] NCCL INFO Channel 01/0 : 3[63000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6a18000ab0 nRanks 96
e08r2n17:22824:23810 [3] NCCL INFO Channel 01/0 : 39[63000] -> 40[4000] [send] via NET/IB/0 comm 0x2ba348000ab0 nRanks 96
e08r3n11:31587:32657 [3] NCCL INFO Channel 01/0 : 95[63000] -> 0[4000] [send] via NET/IB/0 comm 0x2b8dd8000ab0 nRanks 96
e08r3n04:31689:307 [2] NCCL INFO Connected all rings comm 0x2ad454000ab0 nRanks 96 busId 43000
e08r3n03:24232:25250 [2] NCCL INFO Connected all rings comm 0x2acd78000ab0 nRanks 96 busId 43000
e08r2n19:10430:11601 [3] NCCL INFO Channel 01/0 : 47[63000] -> 48[4000] [send] via NET/IB/0 comm 0x2b6930000ab0 nRanks 96
e08r2n18:2638:3717 [3] NCCL INFO Channel 01/0 : 43[63000] -> 44[4000] [send] via NET/IB/0 comm 0x2ac998000ab0 nRanks 96
e08r2n09:28731:29886 [3] NCCL INFO Channel 01/0 : 7[63000] -> 8[4000] [send] via NET/IB/0 comm 0x2b67e8000ab0 nRanks 96
e08r3n00:29330:30333 [3] NCCL INFO Channel 01/0 : 51[63000] -> 52[4000] [send] via NET/IB/0 comm 0x2b0d6c000ab0 nRanks 96
e08r2n16:15467:16571 [3] NCCL INFO Channel 01/0 : 35[63000] -> 36[4000] [send] via NET/IB/0 comm 0x2b1dc0000ab0 nRanks 96
e08r3n04:31689:307 [2] NCCL INFO Channel 00 : 66[43000] -> 65[26000] via P2P/IPC comm 0x2ad454000ab0 nRanks 96
e08r3n03:24232:25250 [2] NCCL INFO Channel 00 : 62[43000] -> 61[26000] via P2P/IPC comm 0x2acd78000ab0 nRanks 96
e08r2n10:2344:3537 [3] NCCL INFO Channel 01/0 : 11[63000] -> 12[4000] [send] via NET/IB/0 comm 0x2b4a18000ab0 nRanks 96
e08r2n13:26096:27117 [3] NCCL INFO Channel 01/0 : 23[63000] -> 24[4000] [send] via NET/IB/0 comm 0x2b83c0000ab0 nRanks 96
e08r2n14:20096:21173 [3] NCCL INFO Channel 01/0 : 27[63000] -> 28[4000] [send] via NET/IB/0 comm 0x2b5e80000ab0 nRanks 96
e08r2n15:23881:25092 [3] NCCL INFO Channel 01/0 : 31[63000] -> 32[4000] [send] via NET/IB/0 comm 0x2b7d9c000ab0 nRanks 96
e08r3n04:31689:307 [2] NCCL INFO Channel 01 : 66[43000] -> 65[26000] via P2P/IPC comm 0x2ad454000ab0 nRanks 96
e08r2n11:7504:8721 [3] NCCL INFO Channel 01/0 : 15[63000] -> 16[4000] [send] via NET/IB/0 comm 0x2baa1c000ab0 nRanks 96
e08r3n03:24232:25250 [2] NCCL INFO Channel 01 : 62[43000] -> 61[26000] via P2P/IPC comm 0x2acd78000ab0 nRanks 96
e08r3n05:26318:27350 [2] NCCL INFO Connected all rings comm 0x2b62cc000ab0 nRanks 96 busId 43000
e08r3n09:10075:11136 [2] NCCL INFO Connected all rings comm 0x2b9dc4000ab0 nRanks 96 busId 43000
e08r3n07:20448:21421 [2] NCCL INFO Connected all rings comm 0x2b4478000ab0 nRanks 96 busId 43000
e08r3n06:24084:25277 [2] NCCL INFO Connected all rings comm 0x2b7908000ab0 nRanks 96 busId 43000
e08r2n12:10511:11620 [3] NCCL INFO Channel 01/0 : 19[63000] -> 20[4000] [send] via NET/IB/0 comm 0x2b3e1c000ab0 nRanks 96
e08r3n08:14730:15934 [2] NCCL INFO Connected all rings comm 0x2aee98000ab0 nRanks 96 busId 43000
e08r3n10:20259:21460 [2] NCCL INFO Connected all rings comm 0x2b837c000ab0 nRanks 96 busId 43000
e08r3n05:26318:27350 [2] NCCL INFO Channel 00 : 70[43000] -> 69[26000] via P2P/IPC comm 0x2b62cc000ab0 nRanks 96
e08r3n09:10075:11136 [2] NCCL INFO Channel 00 : 86[43000] -> 85[26000] via P2P/IPC comm 0x2b9dc4000ab0 nRanks 96
e08r3n01:21695:22709 [2] NCCL INFO Connected all rings comm 0x2ad544000ab0 nRanks 96 busId 43000
e08r3n02:14037:15028 [2] NCCL INFO Connected all rings comm 0x2aecd0000ab0 nRanks 96 busId 43000
e08r3n05:26318:27350 [2] NCCL INFO Channel 01 : 70[43000] -> 69[26000] via P2P/IPC comm 0x2b62cc000ab0 nRanks 96
e08r3n07:20448:21421 [2] NCCL INFO Channel 00 : 78[43000] -> 77[26000] via P2P/IPC comm 0x2b4478000ab0 nRanks 96
e08r3n09:10075:11136 [2] NCCL INFO Channel 01 : 86[43000] -> 85[26000] via P2P/IPC comm 0x2b9dc4000ab0 nRanks 96
e08r3n06:24084:25277 [2] NCCL INFO Channel 00 : 74[43000] -> 73[26000] via P2P/IPC comm 0x2b7908000ab0 nRanks 96
e08r3n10:20259:21460 [2] NCCL INFO Channel 00 : 90[43000] -> 89[26000] via P2P/IPC comm 0x2b837c000ab0 nRanks 96
e08r3n08:14730:15934 [2] NCCL INFO Channel 00 : 82[43000] -> 81[26000] via P2P/IPC comm 0x2aee98000ab0 nRanks 96
e08r2n19:10429:11597 [2] NCCL INFO Connected all rings comm 0x2b019c000ab0 nRanks 96 busId 43000
e08r3n11:31586:32658 [2] NCCL INFO Connected all rings comm 0x2b3058000ab0 nRanks 96 busId 43000
e08r2n08:11396:12836 [2] NCCL INFO Connected all rings comm 0x2b56b4000ab0 nRanks 96 busId 43000
e08r3n07:20448:21421 [2] NCCL INFO Channel 01 : 78[43000] -> 77[26000] via P2P/IPC comm 0x2b4478000ab0 nRanks 96
e08r3n00:29328:30332 [2] NCCL INFO Connected all rings comm 0x2ad580000ab0 nRanks 96 busId 43000
e08r2n17:22823:23812 [2] NCCL INFO Connected all rings comm 0x2af978000ab0 nRanks 96 busId 43000
e08r3n06:24084:25277 [2] NCCL INFO Channel 01 : 74[43000] -> 73[26000] via P2P/IPC comm 0x2b7908000ab0 nRanks 96
e08r3n08:14730:15934 [2] NCCL INFO Channel 01 : 82[43000] -> 81[26000] via P2P/IPC comm 0x2aee98000ab0 nRanks 96
e08r3n02:14037:15028 [2] NCCL INFO Channel 00 : 58[43000] -> 57[26000] via P2P/IPC comm 0x2aecd0000ab0 nRanks 96
e08r3n10:20259:21460 [2] NCCL INFO Channel 01 : 90[43000] -> 89[26000] via P2P/IPC comm 0x2b837c000ab0 nRanks 96
e08r2n18:2637:3714 [2] NCCL INFO Connected all rings comm 0x2b36e4000ab0 nRanks 96 busId 43000
e08r3n01:21695:22709 [2] NCCL INFO Channel 00 : 54[43000] -> 53[26000] via P2P/IPC comm 0x2ad544000ab0 nRanks 96
e08r2n09:28730:29888 [2] NCCL INFO Connected all rings comm 0x2b25f4000ab0 nRanks 96 busId 43000
e08r3n02:14037:15028 [2] NCCL INFO Channel 01 : 58[43000] -> 57[26000] via P2P/IPC comm 0x2aecd0000ab0 nRanks 96
e08r2n19:10429:11597 [2] NCCL INFO Channel 00 : 46[43000] -> 45[26000] via P2P/IPC comm 0x2b019c000ab0 nRanks 96
e08r3n11:31586:32658 [2] NCCL INFO Channel 00 : 94[43000] -> 93[26000] via P2P/IPC comm 0x2b3058000ab0 nRanks 96
e08r2n11:7503:8719 [2] NCCL INFO Connected all rings comm 0x2b4fa4000ab0 nRanks 96 busId 43000
e08r3n01:21695:22709 [2] NCCL INFO Channel 01 : 54[43000] -> 53[26000] via P2P/IPC comm 0x2ad544000ab0 nRanks 96
e08r3n00:29328:30332 [2] NCCL INFO Channel 00 : 50[43000] -> 49[26000] via P2P/IPC comm 0x2ad580000ab0 nRanks 96
e08r2n08:11396:12836 [2] NCCL INFO Channel 00 : 2[43000] -> 1[26000] via P2P/IPC comm 0x2b56b4000ab0 nRanks 96
e08r2n16:15466:16578 [2] NCCL INFO Connected all rings comm 0x2bafec000ab0 nRanks 96 busId 43000
e08r3n11:31586:32658 [2] NCCL INFO Channel 01 : 94[43000] -> 93[26000] via P2P/IPC comm 0x2b3058000ab0 nRanks 96
e08r2n10:2343:3556 [2] NCCL INFO Connected all rings comm 0x2b14cc000ab0 nRanks 96 busId 43000
e08r2n19:10429:11597 [2] NCCL INFO Channel 01 : 46[43000] -> 45[26000] via P2P/IPC comm 0x2b019c000ab0 nRanks 96
e08r2n17:22823:23812 [2] NCCL INFO Channel 00 : 38[43000] -> 37[26000] via P2P/IPC comm 0x2af978000ab0 nRanks 96
e08r2n08:11396:12836 [2] NCCL INFO Channel 01 : 2[43000] -> 1[26000] via P2P/IPC comm 0x2b56b4000ab0 nRanks 96
e08r3n00:29328:30332 [2] NCCL INFO Channel 01 : 50[43000] -> 49[26000] via P2P/IPC comm 0x2ad580000ab0 nRanks 96
e08r2n13:26095:27111 [2] NCCL INFO Connected all rings comm 0x2b1d90000ab0 nRanks 96 busId 43000
e08r2n18:2637:3714 [2] NCCL INFO Channel 00 : 42[43000] -> 41[26000] via P2P/IPC comm 0x2b36e4000ab0 nRanks 96
e08r2n15:23880:25090 [2] NCCL INFO Connected all rings comm 0x2b1de8000ab0 nRanks 96 busId 43000
e08r2n14:20095:21178 [2] NCCL INFO Connected all rings comm 0x2bac40000ab0 nRanks 96 busId 43000
e08r2n09:28730:29888 [2] NCCL INFO Channel 00 : 6[43000] -> 5[26000] via P2P/IPC comm 0x2b25f4000ab0 nRanks 96
e08r2n17:22823:23812 [2] NCCL INFO Channel 01 : 38[43000] -> 37[26000] via P2P/IPC comm 0x2af978000ab0 nRanks 96
e08r2n11:7503:8719 [2] NCCL INFO Channel 00 : 14[43000] -> 13[26000] via P2P/IPC comm 0x2b4fa4000ab0 nRanks 96
e08r2n09:28730:29888 [2] NCCL INFO Channel 01 : 6[43000] -> 5[26000] via P2P/IPC comm 0x2b25f4000ab0 nRanks 96
e08r2n16:15466:16578 [2] NCCL INFO Channel 00 : 34[43000] -> 33[26000] via P2P/IPC comm 0x2bafec000ab0 nRanks 96
e08r2n18:2637:3714 [2] NCCL INFO Channel 01 : 42[43000] -> 41[26000] via P2P/IPC comm 0x2b36e4000ab0 nRanks 96
e08r2n10:2343:3556 [2] NCCL INFO Channel 00 : 10[43000] -> 9[26000] via P2P/IPC comm 0x2b14cc000ab0 nRanks 96
e08r2n11:7503:8719 [2] NCCL INFO Channel 01 : 14[43000] -> 13[26000] via P2P/IPC comm 0x2b4fa4000ab0 nRanks 96
e08r2n16:15466:16578 [2] NCCL INFO Channel 01 : 34[43000] -> 33[26000] via P2P/IPC comm 0x2bafec000ab0 nRanks 96
e08r2n12:10510:11624 [2] NCCL INFO Connected all rings comm 0x2b5f60000ab0 nRanks 96 busId 43000
e08r2n13:26095:27111 [2] NCCL INFO Channel 00 : 22[43000] -> 21[26000] via P2P/IPC comm 0x2b1d90000ab0 nRanks 96
e08r2n10:2343:3556 [2] NCCL INFO Channel 01 : 10[43000] -> 9[26000] via P2P/IPC comm 0x2b14cc000ab0 nRanks 96
e08r2n15:23880:25090 [2] NCCL INFO Channel 00 : 30[43000] -> 29[26000] via P2P/IPC comm 0x2b1de8000ab0 nRanks 96
e08r2n14:20095:21178 [2] NCCL INFO Channel 00 : 26[43000] -> 25[26000] via P2P/IPC comm 0x2bac40000ab0 nRanks 96
e08r2n13:26095:27111 [2] NCCL INFO Channel 01 : 22[43000] -> 21[26000] via P2P/IPC comm 0x2b1d90000ab0 nRanks 96
e08r2n15:23880:25090 [2] NCCL INFO Channel 01 : 30[43000] -> 29[26000] via P2P/IPC comm 0x2b1de8000ab0 nRanks 96
e08r2n14:20095:21178 [2] NCCL INFO Channel 01 : 26[43000] -> 25[26000] via P2P/IPC comm 0x2bac40000ab0 nRanks 96
e08r2n12:10510:11624 [2] NCCL INFO Channel 00 : 18[43000] -> 17[26000] via P2P/IPC comm 0x2b5f60000ab0 nRanks 96
e08r2n12:10510:11624 [2] NCCL INFO Channel 01 : 18[43000] -> 17[26000] via P2P/IPC comm 0x2b5f60000ab0 nRanks 96
e08r3n03:24233:25253 [3] NCCL INFO Connected all rings comm 0x2b66e0000ab0 nRanks 96 busId 63000
e08r3n03:24233:25253 [3] NCCL INFO Channel 00 : 63[63000] -> 62[43000] via P2P/IPC comm 0x2b66e0000ab0 nRanks 96
e08r3n03:24233:25253 [3] NCCL INFO Channel 01 : 63[63000] -> 62[43000] via P2P/IPC comm 0x2b66e0000ab0 nRanks 96
e08r3n04:31690:302 [3] NCCL INFO Connected all rings comm 0x2ac5d0000ab0 nRanks 96 busId 63000
e08r3n04:31690:302 [3] NCCL INFO Channel 00 : 67[63000] -> 66[43000] via P2P/IPC comm 0x2ac5d0000ab0 nRanks 96
e08r3n05:26319:27347 [3] NCCL INFO Connected all rings comm 0x2ae1cc000ab0 nRanks 96 busId 63000
e08r3n05:26319:27347 [3] NCCL INFO Channel 00 : 71[63000] -> 70[43000] via P2P/IPC comm 0x2ae1cc000ab0 nRanks 96
e08r3n03:24233:25253 [3] NCCL INFO Connected all trees comm 0x2b66e0000ab0 nRanks 96 busId 63000
e08r3n04:31690:302 [3] NCCL INFO Channel 01 : 67[63000] -> 66[43000] via P2P/IPC comm 0x2ac5d0000ab0 nRanks 96
e08r3n03:24233:25253 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n03:24233:25253 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n10:20260:21455 [3] NCCL INFO Connected all rings comm 0x2ace2c000ab0 nRanks 96 busId 63000
e08r3n05:26319:27347 [3] NCCL INFO Channel 01 : 71[63000] -> 70[43000] via P2P/IPC comm 0x2ae1cc000ab0 nRanks 96
e08r3n10:20260:21455 [3] NCCL INFO Channel 00 : 91[63000] -> 90[43000] via P2P/IPC comm 0x2ace2c000ab0 nRanks 96
e08r3n09:10076:11135 [3] NCCL INFO Connected all rings comm 0x2ab750000ab0 nRanks 96 busId 63000
e08r3n09:10076:11135 [3] NCCL INFO Channel 00 : 87[63000] -> 86[43000] via P2P/IPC comm 0x2ab750000ab0 nRanks 96
e08r3n10:20260:21455 [3] NCCL INFO Channel 01 : 91[63000] -> 90[43000] via P2P/IPC comm 0x2ace2c000ab0 nRanks 96
e08r3n04:31686:32764 [0] NCCL INFO Connected all rings comm 0x2b08c0000ab0 nRanks 96 busId 4000
e08r3n09:10076:11135 [3] NCCL INFO Channel 01 : 87[63000] -> 86[43000] via P2P/IPC comm 0x2ab750000ab0 nRanks 96
e08r2n08:11397:12837 [3] NCCL INFO Connected all rings comm 0x2b6a18000ab0 nRanks 96 busId 63000
e08r2n08:11397:12837 [3] NCCL INFO Channel 00 : 3[63000] -> 2[43000] via P2P/IPC comm 0x2b6a18000ab0 nRanks 96
e08r2n19:10430:11601 [3] NCCL INFO Connected all rings comm 0x2b6930000ab0 nRanks 96 busId 63000
e08r2n19:10430:11601 [3] NCCL INFO Channel 00 : 47[63000] -> 46[43000] via P2P/IPC comm 0x2b6930000ab0 nRanks 96
e08r3n07:20449:21403 [3] NCCL INFO Connected all rings comm 0x2b2570000ab0 nRanks 96 busId 63000
e08r2n14:20096:21173 [3] NCCL INFO Connected all rings comm 0x2b5e80000ab0 nRanks 96 busId 63000
e08r2n08:11397:12837 [3] NCCL INFO Channel 01 : 3[63000] -> 2[43000] via P2P/IPC comm 0x2b6a18000ab0 nRanks 96
e08r3n07:20449:21403 [3] NCCL INFO Channel 00 : 79[63000] -> 78[43000] via P2P/IPC comm 0x2b2570000ab0 nRanks 96
e08r2n14:20096:21173 [3] NCCL INFO Channel 00 : 27[63000] -> 26[43000] via P2P/IPC comm 0x2b5e80000ab0 nRanks 96
e08r2n09:28731:29886 [3] NCCL INFO Connected all rings comm 0x2b67e8000ab0 nRanks 96 busId 63000
e08r2n09:28731:29886 [3] NCCL INFO Channel 00 : 7[63000] -> 6[43000] via P2P/IPC comm 0x2b67e8000ab0 nRanks 96
e08r3n06:24085:25276 [3] NCCL INFO Connected all rings comm 0x2b6f04000ab0 nRanks 96 busId 63000
e08r2n18:2638:3717 [3] NCCL INFO Connected all rings comm 0x2ac998000ab0 nRanks 96 busId 63000
e08r3n06:24085:25276 [3] NCCL INFO Channel 00 : 75[63000] -> 74[43000] via P2P/IPC comm 0x2b6f04000ab0 nRanks 96
e08r2n18:2638:3717 [3] NCCL INFO Channel 00 : 43[63000] -> 42[43000] via P2P/IPC comm 0x2ac998000ab0 nRanks 96
e08r2n19:10430:11601 [3] NCCL INFO Channel 01 : 47[63000] -> 46[43000] via P2P/IPC comm 0x2b6930000ab0 nRanks 96
e08r3n01:21696:22700 [3] NCCL INFO Connected all rings comm 0x2b9c1c000ab0 nRanks 96 busId 63000
e08r3n01:21696:22700 [3] NCCL INFO Channel 00 : 55[63000] -> 54[43000] via P2P/IPC comm 0x2b9c1c000ab0 nRanks 96
e08r2n14:20096:21173 [3] NCCL INFO Channel 01 : 27[63000] -> 26[43000] via P2P/IPC comm 0x2b5e80000ab0 nRanks 96
e08r3n11:31587:32657 [3] NCCL INFO Connected all rings comm 0x2b8dd8000ab0 nRanks 96 busId 63000
e08r2n09:28731:29886 [3] NCCL INFO Channel 01 : 7[63000] -> 6[43000] via P2P/IPC comm 0x2b67e8000ab0 nRanks 96
e08r3n07:20449:21403 [3] NCCL INFO Channel 01 : 79[63000] -> 78[43000] via P2P/IPC comm 0x2b2570000ab0 nRanks 96
e08r3n11:31587:32657 [3] NCCL INFO Channel 00 : 95[63000] -> 94[43000] via P2P/IPC comm 0x2b8dd8000ab0 nRanks 96
e08r2n16:15467:16571 [3] NCCL INFO Connected all rings comm 0x2b1dc0000ab0 nRanks 96 busId 63000
e08r3n06:24085:25276 [3] NCCL INFO Channel 01 : 75[63000] -> 74[43000] via P2P/IPC comm 0x2b6f04000ab0 nRanks 96
e08r2n17:22824:23810 [3] NCCL INFO Connected all rings comm 0x2ba348000ab0 nRanks 96 busId 63000
e08r2n16:15467:16571 [3] NCCL INFO Channel 00 : 35[63000] -> 34[43000] via P2P/IPC comm 0x2b1dc0000ab0 nRanks 96
e08r2n17:22824:23810 [3] NCCL INFO Channel 00 : 39[63000] -> 38[43000] via P2P/IPC comm 0x2ba348000ab0 nRanks 96
e08r2n18:2638:3717 [3] NCCL INFO Channel 01 : 43[63000] -> 42[43000] via P2P/IPC comm 0x2ac998000ab0 nRanks 96
e08r3n01:21696:22700 [3] NCCL INFO Channel 01 : 55[63000] -> 54[43000] via P2P/IPC comm 0x2b9c1c000ab0 nRanks 96
e08r3n11:31587:32657 [3] NCCL INFO Channel 01 : 95[63000] -> 94[43000] via P2P/IPC comm 0x2b8dd8000ab0 nRanks 96
e08r3n08:14731:15932 [3] NCCL INFO Connected all rings comm 0x2b99d8000ab0 nRanks 96 busId 63000
e08r3n08:14731:15932 [3] NCCL INFO Channel 00 : 83[63000] -> 82[43000] via P2P/IPC comm 0x2b99d8000ab0 nRanks 96
e08r3n02:14038:15023 [3] NCCL INFO Connected all rings comm 0x2b2d40000ab0 nRanks 96 busId 63000
e08r2n11:7504:8721 [3] NCCL INFO Connected all rings comm 0x2baa1c000ab0 nRanks 96 busId 63000
e08r3n02:14038:15023 [3] NCCL INFO Channel 00 : 59[63000] -> 58[43000] via P2P/IPC comm 0x2b2d40000ab0 nRanks 96
e08r2n11:7504:8721 [3] NCCL INFO Channel 00 : 15[63000] -> 14[43000] via P2P/IPC comm 0x2baa1c000ab0 nRanks 96
e08r2n16:15467:16571 [3] NCCL INFO Channel 01 : 35[63000] -> 34[43000] via P2P/IPC comm 0x2b1dc0000ab0 nRanks 96
e08r2n17:22824:23810 [3] NCCL INFO Channel 01 : 39[63000] -> 38[43000] via P2P/IPC comm 0x2ba348000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Connected all rings comm 0x2b7410000ab0 nRanks 96 busId 4000
e08r3n10:20256:21447 [0] NCCL INFO Connected all rings comm 0x2b6488000ab0 nRanks 96 busId 4000
e08r3n00:29330:30333 [3] NCCL INFO Connected all rings comm 0x2b0d6c000ab0 nRanks 96 busId 63000
e08r3n00:29330:30333 [3] NCCL INFO Channel 00 : 51[63000] -> 50[43000] via P2P/IPC comm 0x2b0d6c000ab0 nRanks 96
e08r3n05:26319:27347 [3] NCCL INFO Connected all trees comm 0x2ae1cc000ab0 nRanks 96 busId 63000
e08r3n08:14731:15932 [3] NCCL INFO Channel 01 : 83[63000] -> 82[43000] via P2P/IPC comm 0x2b99d8000ab0 nRanks 96
e08r3n05:26319:27347 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n05:26319:27347 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14038:15023 [3] NCCL INFO Channel 01 : 59[63000] -> 58[43000] via P2P/IPC comm 0x2b2d40000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Connected all rings comm 0x2b280c000ab0 nRanks 96 busId 4000
e08r3n10:20260:21455 [3] NCCL INFO Connected all trees comm 0x2ace2c000ab0 nRanks 96 busId 63000
e08r2n11:7504:8721 [3] NCCL INFO Channel 01 : 15[63000] -> 14[43000] via P2P/IPC comm 0x2baa1c000ab0 nRanks 96
e08r2n08:11393:12596 [0] NCCL INFO Connected all rings comm 0x2afee0000ab0 nRanks 96 busId 4000
e08r2n13:26096:27117 [3] NCCL INFO Connected all rings comm 0x2b83c0000ab0 nRanks 96 busId 63000
e08r3n11:31583:32648 [0] NCCL INFO Connected all rings comm 0x2af294000ab0 nRanks 96 busId 4000
e08r3n00:29330:30333 [3] NCCL INFO Channel 01 : 51[63000] -> 50[43000] via P2P/IPC comm 0x2b0d6c000ab0 nRanks 96
e08r2n13:26096:27117 [3] NCCL INFO Channel 00 : 23[63000] -> 22[43000] via P2P/IPC comm 0x2b83c0000ab0 nRanks 96
e08r3n10:20260:21455 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n10:20260:21455 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2344:3537 [3] NCCL INFO Connected all rings comm 0x2b4a18000ab0 nRanks 96 busId 63000
e08r2n10:2344:3537 [3] NCCL INFO Channel 00 : 11[63000] -> 10[43000] via P2P/IPC comm 0x2b4a18000ab0 nRanks 96
e08r3n09:10076:11135 [3] NCCL INFO Connected all trees comm 0x2ab750000ab0 nRanks 96 busId 63000
e08r2n13:26096:27117 [3] NCCL INFO Channel 01 : 23[63000] -> 22[43000] via P2P/IPC comm 0x2b83c0000ab0 nRanks 96
e08r3n09:10076:11135 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n09:10076:11135 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n07:20445:21399 [0] NCCL INFO Connected all rings comm 0x2ab264000ab0 nRanks 96 busId 4000
e08r2n10:2344:3537 [3] NCCL INFO Channel 01 : 11[63000] -> 10[43000] via P2P/IPC comm 0x2b4a18000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Connected all rings comm 0x2aff70000ab0 nRanks 96 busId 4000
e08r3n02:14034:15017 [0] NCCL INFO Connected all rings comm 0x2b57b8000ab0 nRanks 96 busId 4000
e08r2n08:11397:12837 [3] NCCL INFO Connected all trees comm 0x2b6a18000ab0 nRanks 96 busId 63000
e08r2n08:11397:12837 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11397:12837 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10511:11620 [3] NCCL INFO Connected all rings comm 0x2b3e1c000ab0 nRanks 96 busId 63000
e08r3n08:14727:15954 [0] NCCL INFO Connected all rings comm 0x2ac6a8000ab0 nRanks 96 busId 4000
e08r2n12:10511:11620 [3] NCCL INFO Channel 00 : 19[63000] -> 18[43000] via P2P/IPC comm 0x2b3e1c000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Connected all rings comm 0x2af198000ab0 nRanks 96 busId 4000
e08r2n09:28728:29878 [0] NCCL INFO Connected all rings comm 0x2af210000ab0 nRanks 96 busId 4000
e08r3n03:24229:25255 [0] NCCL INFO Connected all rings comm 0x2ba43c000ab0 nRanks 96 busId 4000
e08r2n19:10430:11601 [3] NCCL INFO Connected all trees comm 0x2b6930000ab0 nRanks 96 busId 63000
e08r2n08:11396:12836 [2] NCCL INFO Connected all trees comm 0x2b56b4000ab0 nRanks 96 busId 43000
e08r2n08:11396:12836 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11396:12836 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10430:11601 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n19:10430:11601 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2340:3530 [0] NCCL INFO Connected all rings comm 0x2b76f0000ab0 nRanks 96 busId 4000
e08r3n07:20449:21403 [3] NCCL INFO Connected all trees comm 0x2b2570000ab0 nRanks 96 busId 63000
e08r2n14:20096:21173 [3] NCCL INFO Connected all trees comm 0x2b5e80000ab0 nRanks 96 busId 63000
e08r3n07:20449:21403 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n14:20096:21173 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n14:20096:21173 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n18:2634:3704 [0] NCCL INFO Connected all rings comm 0x2ac970000ab0 nRanks 96 busId 4000
e08r2n17:22820:23808 [0] NCCL INFO Connected all rings comm 0x2b02c0000ab0 nRanks 96 busId 4000
e08r2n15:23881:25092 [3] NCCL INFO Connected all rings comm 0x2b7d9c000ab0 nRanks 96 busId 63000
e08r3n09:10072:11133 [0] NCCL INFO Connected all rings comm 0x2ae258000ab0 nRanks 96 busId 4000
e08r3n07:20449:21403 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n15:23881:25092 [3] NCCL INFO Channel 00 : 31[63000] -> 30[43000] via P2P/IPC comm 0x2b7d9c000ab0 nRanks 96
e08r3n06:24085:25276 [3] NCCL INFO Connected all trees comm 0x2b6f04000ab0 nRanks 96 busId 63000
e08r3n06:24085:25276 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2638:3717 [3] NCCL INFO Connected all trees comm 0x2ac998000ab0 nRanks 96 busId 63000
e08r3n01:21696:22700 [3] NCCL INFO Connected all trees comm 0x2b9c1c000ab0 nRanks 96 busId 63000
e08r2n12:10511:11620 [3] NCCL INFO Channel 01 : 19[63000] -> 18[43000] via P2P/IPC comm 0x2b3e1c000ab0 nRanks 96
e08r3n06:24085:25276 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n18:2638:3717 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2638:3717 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n01:21696:22700 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23877:25081 [0] NCCL INFO Connected all rings comm 0x2b227c000ab0 nRanks 96 busId 4000
e08r3n01:21696:22700 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n15:23881:25092 [3] NCCL INFO Channel 01 : 31[63000] -> 30[43000] via P2P/IPC comm 0x2b7d9c000ab0 nRanks 96
e08r2n17:22824:23810 [3] NCCL INFO Connected all trees comm 0x2ba348000ab0 nRanks 96 busId 63000
e08r2n16:15467:16571 [3] NCCL INFO Connected all trees comm 0x2b1dc0000ab0 nRanks 96 busId 63000
e08r2n17:22824:23810 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n17:22824:23810 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15467:16571 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n16:15467:16571 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n11:31587:32657 [3] NCCL INFO Connected all trees comm 0x2b8dd8000ab0 nRanks 96 busId 63000
e08r2n12:10508:11619 [0] NCCL INFO Connected all rings comm 0x2b625c000ab0 nRanks 96 busId 4000
e08r2n09:28731:29886 [3] NCCL INFO Connected all trees comm 0x2b67e8000ab0 nRanks 96 busId 63000
e08r3n11:31587:32657 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n11:31587:32657 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n09:28731:29886 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n01:21692:22698 [0] NCCL INFO Connected all rings comm 0x2accfc000ab0 nRanks 96 busId 4000
e08r3n11:31586:32658 [2] NCCL INFO Connected all trees comm 0x2b3058000ab0 nRanks 96 busId 43000
e08r2n09:28731:29886 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n08:14731:15932 [3] NCCL INFO Connected all trees comm 0x2b99d8000ab0 nRanks 96 busId 63000
e08r3n02:14038:15023 [3] NCCL INFO Connected all trees comm 0x2b2d40000ab0 nRanks 96 busId 63000
e08r2n14:20092:21169 [0] NCCL INFO Connected all rings comm 0x2aec8c000ab0 nRanks 96 busId 4000
e08r3n11:31586:32658 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14731:15932 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14731:15932 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14038:15023 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n11:31586:32658 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14038:15023 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7504:8721 [3] NCCL INFO Connected all trees comm 0x2baa1c000ab0 nRanks 96 busId 63000
e08r2n11:7504:8721 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n11:7504:8721 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n00:29330:30333 [3] NCCL INFO Connected all trees comm 0x2b0d6c000ab0 nRanks 96 busId 63000
e08r3n00:29330:30333 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26093:27103 [0] NCCL INFO Connected all rings comm 0x2b19ac000ab0 nRanks 96 busId 4000
e08r3n00:29330:30333 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26096:27117 [3] NCCL INFO Connected all trees comm 0x2b83c0000ab0 nRanks 96 busId 63000
e08r2n13:26096:27117 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26096:27117 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7501:8711 [0] NCCL INFO Connected all rings comm 0x2b6d8c000ab0 nRanks 96 busId 4000
e08r2n10:2344:3537 [3] NCCL INFO Connected all trees comm 0x2b4a18000ab0 nRanks 96 busId 63000
e08r2n10:2344:3537 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2344:3537 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15463:16564 [0] NCCL INFO Connected all rings comm 0x2af018000ab0 nRanks 96 busId 4000
e08r2n12:10511:11620 [3] NCCL INFO Connected all trees comm 0x2b3e1c000ab0 nRanks 96 busId 63000
e08r2n12:10511:11620 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10511:11620 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n15:23881:25092 [3] NCCL INFO Connected all trees comm 0x2b7d9c000ab0 nRanks 96 busId 63000
e08r2n15:23881:25092 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23881:25092 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31690:302 [3] NCCL INFO Connected all trees comm 0x2ac5d0000ab0 nRanks 96 busId 63000
e08r3n04:31690:302 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n04:31690:302 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31686:32764 [0] NCCL INFO Channel 01/0 : 64[4000] -> 68[4000] [send] via NET/IB/0 comm 0x2b08c0000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 01/0 : 69[26000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b280c000ab0 nRanks 96
e08r3n10:20256:21447 [0] NCCL INFO Channel 01/0 : 85[26000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b6488000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 01/0 : 64[4000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7410000ab0 nRanks 96
e08r2n08:11393:12596 [0] NCCL INFO Channel 01/0 : 0[4000] -> 4[4000] [send] via NET/IB/0 comm 0x2afee0000ab0 nRanks 96
e08r3n11:31583:32648 [0] NCCL INFO Channel 00/0 : 88[4000] -> 92[4000] [receive] via NET/IB/0 comm 0x2af294000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 00/0 : 72[4000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab264000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 00/0 : 40[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2aff70000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 01/0 : 48[4000] -> 52[4000] [send] via NET/IB/0 comm 0x2af198000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 01/0 : 0[4000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af210000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 01/0 : 53[26000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b57b8000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 01/0 : 37[26000] -> 40[4000] [receive] via NET/IB/0 comm 0x2ac970000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 00/0 : 56[4000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba43c000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 01/0 : 80[4000] -> 84[4000] [send] via NET/IB/0 comm 0x2ac6a8000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 01/0 : 5[26000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b76f0000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 01/0 : 80[4000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae258000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Channel 01/0 : 32[4000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b02c0000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 00/0 : 24[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b227c000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 01/0 : 48[4000] -> 52[4000] [receive] via NET/IB/0 comm 0x2accfc000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 01/0 : 16[4000] -> 20[4000] [send] via NET/IB/0 comm 0x2b625c000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 01/0 : 16[4000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b19ac000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 01/0 : 21[26000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aec8c000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 00/0 : 8[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6d8c000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 01/0 : 32[4000] -> 36[4000] [send] via NET/IB/0 comm 0x2af018000ab0 nRanks 96
e08r3n09:10074:11130 [1] NCCL INFO Channel 01/0 : 88[4000] -> 85[26000] [receive] via NET/IB/0 comm 0x2aed0c000ab0 nRanks 96
e08r3n04:31686:32764 [0] NCCL INFO Channel 00/0 : 64[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b08c0000ab0 nRanks 96
e08r3n05:26317:27342 [1] NCCL INFO Channel 01/0 : 72[4000] -> 69[26000] [receive] via NET/IB/0 comm 0x2b7c88000ab0 nRanks 96
e08r2n08:11393:12596 [0] NCCL INFO Channel 00/0 : 64[4000] -> 0[4000] [receive] via NET/IB/0 comm 0x2afee0000ab0 nRanks 96
e08r3n01:21694:22707 [1] NCCL INFO Channel 01/0 : 56[4000] -> 53[26000] [receive] via NET/IB/0 comm 0x2b91cc000ab0 nRanks 96
e08r2n09:28729:29883 [1] NCCL INFO Channel 01/0 : 8[4000] -> 5[26000] [receive] via NET/IB/0 comm 0x2b6e70000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 00/0 : 80[4000] -> 88[4000] [send] via NET/IB/0 comm 0x2ac6a8000ab0 nRanks 96
e08r2n17:22822:23809 [1] NCCL INFO Channel 01/0 : 40[4000] -> 37[26000] [receive] via NET/IB/0 comm 0x2b7c0c000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 00/0 : 48[4000] -> 56[4000] [send] via NET/IB/0 comm 0x2af198000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 00/0 : 16[4000] -> 24[4000] [send] via NET/IB/0 comm 0x2b625c000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 00/0 : 68[4000] -> 73[26000] [send] via NET/IB/0 comm 0x2b7410000ab0 nRanks 96
e08r2n13:26094:27113 [1] NCCL INFO Channel 01/0 : 24[4000] -> 21[26000] [receive] via NET/IB/0 comm 0x2b2d08000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 00/0 : 4[4000] -> 9[26000] [send] via NET/IB/0 comm 0x2af210000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 00/0 : 32[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2af018000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 00/0 : 84[4000] -> 89[26000] [send] via NET/IB/0 comm 0x2ae258000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 00/0 : 72[4000] -> 76[4000] [send] via NET/IB/0 comm 0x2b280c000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 00/0 : 52[4000] -> 57[26000] [send] via NET/IB/0 comm 0x2accfc000ab0 nRanks 96
e08r3n10:20256:21447 [0] NCCL INFO Channel 00/0 : 88[4000] -> 92[4000] [send] via NET/IB/0 comm 0x2b6488000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 00/0 : 20[4000] -> 25[26000] [send] via NET/IB/0 comm 0x2b19ac000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 00/0 : 40[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2ac970000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 00/0 : 56[4000] -> 60[4000] [send] via NET/IB/0 comm 0x2b57b8000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 00/0 : 8[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2b76f0000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Channel 00/0 : 36[4000] -> 41[26000] [send] via NET/IB/0 comm 0x2b02c0000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 00/0 : 24[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2aec8c000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 01/0 : 68[4000] -> 76[4000] [send] via NET/IB/0 comm 0x2b7410000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 01/0 : 4[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2af210000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 01/0 : 77[26000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae258000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 00/0 : 72[4000] -> 81[26000] [send] via NET/IB/0 comm 0x2b280c000ab0 nRanks 96
e08r3n10:20256:21447 [0] NCCL INFO Channel 00/0 : 80[4000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b6488000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 01/0 : 45[26000] -> 52[4000] [receive] via NET/IB/0 comm 0x2accfc000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 00/0 : 40[4000] -> 49[26000] [send] via NET/IB/0 comm 0x2ac970000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 00/0 : 48[4000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b57b8000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 00/0 : 8[4000] -> 17[26000] [send] via NET/IB/0 comm 0x2b76f0000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 01/0 : 13[26000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b19ac000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Channel 01/0 : 36[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2b02c0000ab0 nRanks 96
e08r3n06:24083:25278 [1] NCCL INFO Channel 00/0 : 73[26000] -> 68[4000] [send] via NET/IB/0 comm 0x2acfe4000ab0 nRanks 96
e08r3n10:20258:21448 [1] NCCL INFO Channel 00/0 : 89[26000] -> 84[4000] [send] via NET/IB/0 comm 0x2b129c000ab0 nRanks 96
e08r2n10:2342:3554 [1] NCCL INFO Channel 00/0 : 9[26000] -> 4[4000] [send] via NET/IB/0 comm 0x2ac06c000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 00/0 : 16[4000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aec8c000ab0 nRanks 96
e08r3n02:14036:15025 [1] NCCL INFO Channel 00/0 : 57[26000] -> 52[4000] [send] via NET/IB/0 comm 0x2b03ec000ab0 nRanks 96
e08r2n14:20094:21174 [1] NCCL INFO Channel 00/0 : 25[26000] -> 20[4000] [send] via NET/IB/0 comm 0x2b550c000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 01/0 : 68[4000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab264000ab0 nRanks 96
e08r2n18:2636:3716 [1] NCCL INFO Channel 00/0 : 41[26000] -> 36[4000] [send] via NET/IB/0 comm 0x2ac578000ab0 nRanks 96
e08r3n11:31583:32648 [0] NCCL INFO Channel 01/0 : 92[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2af294000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 01/0 : 36[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2aff70000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 01/0 : 44[4000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba43c000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 01/0 : 4[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6d8c000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 01/0 : 12[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b227c000ab0 nRanks 96
e08r3n07:20447:21419 [1] NCCL INFO Channel 01/0 : 84[4000] -> 77[26000] [receive] via NET/IB/0 comm 0x2b8d08000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 00/0 : 81[26000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b280c000ab0 nRanks 96
e08r2n19:10428:11592 [1] NCCL INFO Channel 01/0 : 52[4000] -> 45[26000] [receive] via NET/IB/0 comm 0x2b3348000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 00/0 : 49[26000] -> 40[4000] [receive] via NET/IB/0 comm 0x2ac970000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 00/0 : 32[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af198000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 00/0 : 64[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac6a8000ab0 nRanks 96
e08r2n11:7502:8716 [1] NCCL INFO Channel 01/0 : 20[4000] -> 13[26000] [receive] via NET/IB/0 comm 0x2b7cdc000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 00/0 : 17[26000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b76f0000ab0 nRanks 96
e08r3n08:14729:15948 [1] NCCL INFO Channel 00/0 : 81[26000] -> 72[4000] [send] via NET/IB/0 comm 0x2b1290000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 01/0 : 84[4000] -> 77[26000] [send] via NET/IB/0 comm 0x2ae258000ab0 nRanks 96
e08r3n10:20256:21447 [0] NCCL INFO Channel 00/0 : 88[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b6488000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 01/0 : 52[4000] -> 45[26000] [send] via NET/IB/0 comm 0x2accfc000ab0 nRanks 96
e08r3n00:29327:30330 [1] NCCL INFO Channel 00/0 : 49[26000] -> 40[4000] [send] via NET/IB/0 comm 0x2b149c000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 00/0 : 16[4000] -> 33[26000] [send] via NET/IB/0 comm 0x2b625c000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 01/0 : 76[4000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7410000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 00/0 : 56[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2b57b8000ab0 nRanks 96
e08r2n12:10509:11625 [1] NCCL INFO Channel 00/0 : 17[26000] -> 8[4000] [send] via NET/IB/0 comm 0x2bacb8000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 01/0 : 20[4000] -> 13[26000] [send] via NET/IB/0 comm 0x2b19ac000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 01/0 : 61[26000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab264000ab0 nRanks 96
e08r2n17:22820:23808 [0] NCCL INFO Channel 01/0 : 44[4000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b02c0000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 01/0 : 12[4000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af210000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 01/0 : 12[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2b6d8c000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 01/0 : 44[4000] -> 60[4000] [send] via NET/IB/0 comm 0x2aff70000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 00/0 : 24[4000] -> 16[4000] [send] via NET/IB/0 comm 0x2aec8c000ab0 nRanks 96
e08r3n08:14729:15948 [1] NCCL INFO Channel 00 : 81[26000] -> 80[4000] via P2P/IPC comm 0x2b1290000ab0 nRanks 96
e08r3n08:14729:15948 [1] NCCL INFO Channel 01 : 81[26000] -> 80[4000] via P2P/IPC comm 0x2b1290000ab0 nRanks 96
e08r3n00:29327:30330 [1] NCCL INFO Channel 00 : 49[26000] -> 48[4000] via P2P/IPC comm 0x2b149c000ab0 nRanks 96
e08r3n00:29327:30330 [1] NCCL INFO Channel 01 : 49[26000] -> 48[4000] via P2P/IPC comm 0x2b149c000ab0 nRanks 96
e08r2n12:10509:11625 [1] NCCL INFO Channel 00 : 17[26000] -> 16[4000] via P2P/IPC comm 0x2bacb8000ab0 nRanks 96
e08r2n12:10509:11625 [1] NCCL INFO Channel 01 : 17[26000] -> 16[4000] via P2P/IPC comm 0x2bacb8000ab0 nRanks 96
e08r3n08:14730:15934 [2] NCCL INFO Connected all trees comm 0x2aee98000ab0 nRanks 96 busId 43000
e08r3n08:14730:15934 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14730:15934 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n00:29328:30332 [2] NCCL INFO Connected all trees comm 0x2ad580000ab0 nRanks 96 busId 43000
e08r3n00:29328:30332 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n00:29328:30332 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10510:11624 [2] NCCL INFO Connected all trees comm 0x2b5f60000ab0 nRanks 96 busId 43000
e08r2n19:10428:11592 [1] NCCL INFO Channel 00 : 45[26000] -> 44[4000] via P2P/IPC comm 0x2b3348000ab0 nRanks 96
e08r2n12:10510:11624 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10510:11624 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10428:11592 [1] NCCL INFO Channel 01 : 45[26000] -> 44[4000] via P2P/IPC comm 0x2b3348000ab0 nRanks 96
e08r2n11:7502:8716 [1] NCCL INFO Channel 00 : 13[26000] -> 12[4000] via P2P/IPC comm 0x2b7cdc000ab0 nRanks 96
e08r2n11:7502:8716 [1] NCCL INFO Channel 01 : 13[26000] -> 12[4000] via P2P/IPC comm 0x2b7cdc000ab0 nRanks 96
e08r3n07:20447:21419 [1] NCCL INFO Channel 00 : 77[26000] -> 76[4000] via P2P/IPC comm 0x2b8d08000ab0 nRanks 96
e08r3n07:20447:21419 [1] NCCL INFO Channel 01 : 77[26000] -> 76[4000] via P2P/IPC comm 0x2b8d08000ab0 nRanks 96
e08r2n19:10429:11597 [2] NCCL INFO Connected all trees comm 0x2b019c000ab0 nRanks 96 busId 43000
e08r2n19:10429:11597 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n19:10429:11597 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7503:8719 [2] NCCL INFO Connected all trees comm 0x2b4fa4000ab0 nRanks 96 busId 43000
e08r2n11:7503:8719 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n11:7503:8719 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n07:20448:21421 [2] NCCL INFO Connected all trees comm 0x2b4478000ab0 nRanks 96 busId 43000
e08r3n07:20448:21421 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n07:20448:21421 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31686:32764 [0] NCCL INFO Channel 00/0 : 64[4000] -> 0[4000] [send] via NET/IB/0 comm 0x2b08c0000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 00/0 : 32[4000] -> 65[26000] [send] via NET/IB/0 comm 0x2af018000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 00/0 : 89[26000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae258000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 00/0 : 57[26000] -> 52[4000] [receive] via NET/IB/0 comm 0x2accfc000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 00/0 : 33[26000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b625c000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 00/0 : 80[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2ac6a8000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 00/0 : 25[26000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b19ac000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 00/0 : 48[4000] -> 32[4000] [send] via NET/IB/0 comm 0x2af198000ab0 nRanks 96
e08r3n03:24231:25246 [1] NCCL INFO Channel 01/0 : 76[4000] -> 61[26000] [receive] via NET/IB/0 comm 0x2b3040000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 01/0 : 28[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6d8c000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 00/0 : 76[4000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b280c000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 01/0 : 60[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2aff70000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 00/0 : 44[4000] -> 40[4000] [receive] via NET/IB/0 comm 0x2ac970000ab0 nRanks 96
e08r2n16:15465:16568 [1] NCCL INFO Channel 00/0 : 33[26000] -> 16[4000] [send] via NET/IB/0 comm 0x2ad1a8000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 00/0 : 12[4000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b76f0000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 01/0 : 76[4000] -> 61[26000] [send] via NET/IB/0 comm 0x2ab264000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 01/0 : 29[26000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba43c000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 01/0 : 92[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b227c000ab0 nRanks 96
e08r3n10:20258:21448 [1] NCCL INFO Channel 00 : 89[26000] -> 88[4000] via P2P/IPC comm 0x2b129c000ab0 nRanks 96
e08r3n10:20258:21448 [1] NCCL INFO Channel 01 : 89[26000] -> 88[4000] via P2P/IPC comm 0x2b129c000ab0 nRanks 96
e08r3n02:14036:15025 [1] NCCL INFO Channel 00 : 57[26000] -> 56[4000] via P2P/IPC comm 0x2b03ec000ab0 nRanks 96
e08r3n02:14036:15025 [1] NCCL INFO Channel 01 : 57[26000] -> 56[4000] via P2P/IPC comm 0x2b03ec000ab0 nRanks 96
e08r2n14:20094:21174 [1] NCCL INFO Channel 00 : 25[26000] -> 24[4000] via P2P/IPC comm 0x2b550c000ab0 nRanks 96
e08r2n14:20094:21174 [1] NCCL INFO Channel 01 : 25[26000] -> 24[4000] via P2P/IPC comm 0x2b550c000ab0 nRanks 96
e08r3n10:20259:21460 [2] NCCL INFO Connected all trees comm 0x2b837c000ab0 nRanks 96 busId 43000
e08r3n10:20259:21460 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n10:20259:21460 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15465:16568 [1] NCCL INFO Channel 00 : 33[26000] -> 32[4000] via P2P/IPC comm 0x2ad1a8000ab0 nRanks 96
e08r3n02:14037:15028 [2] NCCL INFO Connected all trees comm 0x2aecd0000ab0 nRanks 96 busId 43000
e08r2n16:15465:16568 [1] NCCL INFO Channel 01 : 33[26000] -> 32[4000] via P2P/IPC comm 0x2ad1a8000ab0 nRanks 96
e08r3n02:14037:15028 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n02:14037:15028 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20095:21178 [2] NCCL INFO Connected all trees comm 0x2bac40000ab0 nRanks 96 busId 43000
e08r2n14:20095:21178 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n14:20095:21178 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15466:16578 [2] NCCL INFO Connected all trees comm 0x2bafec000ab0 nRanks 96 busId 43000
e08r2n16:15466:16578 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n16:15466:16578 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n03:24231:25246 [1] NCCL INFO Channel 00 : 61[26000] -> 60[4000] via P2P/IPC comm 0x2b3040000ab0 nRanks 96
e08r3n03:24231:25246 [1] NCCL INFO Channel 01 : 61[26000] -> 60[4000] via P2P/IPC comm 0x2b3040000ab0 nRanks 96
e08r3n03:24232:25250 [2] NCCL INFO Connected all trees comm 0x2acd78000ab0 nRanks 96 busId 43000
e08r3n03:24232:25250 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n03:24232:25250 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31686:32764 [0] NCCL INFO Channel 00/0 : 0[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b08c0000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 00/0 : 65[26000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af018000ab0 nRanks 96
e08r3n04:31688:304 [1] NCCL INFO Channel 00/0 : 65[26000] -> 32[4000] [send] via NET/IB/0 comm 0x2af228000ab0 nRanks 96
e08r2n08:11393:12596 [0] NCCL INFO Channel 00/0 : 0[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2afee0000ab0 nRanks 96
e08r3n09:10072:11133 [0] NCCL INFO Channel 01/0 : 84[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2ae258000ab0 nRanks 96
e08r3n01:21692:22698 [0] NCCL INFO Channel 01/0 : 52[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2accfc000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 01/0 : 76[4000] -> 68[4000] [send] via NET/IB/0 comm 0x2ab264000ab0 nRanks 96
e08r2n13:26093:27103 [0] NCCL INFO Channel 01/0 : 20[4000] -> 16[4000] [send] via NET/IB/0 comm 0x2b19ac000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 00/0 : 24[4000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b625c000ab0 nRanks 96
e08r2n15:23879:25082 [1] NCCL INFO Channel 01/0 : 60[4000] -> 29[26000] [receive] via NET/IB/0 comm 0x2b0918000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 01/0 : 60[4000] -> 29[26000] [send] via NET/IB/0 comm 0x2ba43c000ab0 nRanks 96
e08r3n11:31583:32648 [0] NCCL INFO Channel 01/0 : 28[4000] -> 92[4000] [receive] via NET/IB/0 comm 0x2af294000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 01/0 : 28[4000] -> 92[4000] [send] via NET/IB/0 comm 0x2b227c000ab0 nRanks 96
e08r3n04:31688:304 [1] NCCL INFO Channel 00 : 65[26000] -> 64[4000] via P2P/IPC comm 0x2af228000ab0 nRanks 96
e08r3n04:31688:304 [1] NCCL INFO Channel 01 : 65[26000] -> 64[4000] via P2P/IPC comm 0x2af228000ab0 nRanks 96
e08r3n04:31689:307 [2] NCCL INFO Connected all trees comm 0x2ad454000ab0 nRanks 96 busId 43000
e08r3n04:31689:307 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n04:31689:307 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n15:23879:25082 [1] NCCL INFO Channel 00 : 29[26000] -> 28[4000] via P2P/IPC comm 0x2b0918000ab0 nRanks 96
e08r2n15:23879:25082 [1] NCCL INFO Channel 01 : 29[26000] -> 28[4000] via P2P/IPC comm 0x2b0918000ab0 nRanks 96
e08r2n15:23880:25090 [2] NCCL INFO Connected all trees comm 0x2b1de8000ab0 nRanks 96 busId 43000
e08r2n15:23880:25090 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23880:25090 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n08:11393:12596 [0] NCCL INFO Channel 01/0 : 4[4000] -> 0[4000] [receive] via NET/IB/0 comm 0x2afee0000ab0 nRanks 96
e08r3n04:31686:32764 [0] NCCL INFO Channel 00/0 : 80[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b08c0000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 00/0 : 28[4000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aec8c000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Channel 00/0 : 76[4000] -> 72[4000] [send] via NET/IB/0 comm 0x2ab264000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 00/0 : 48[4000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af018000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 01/0 : 60[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2ba43c000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 00/0 : 73[26000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7410000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 01/0 : 28[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2b227c000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO Channel 01/0 : 20[4000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b625c000ab0 nRanks 96
e08r3n11:31583:32648 [0] NCCL INFO Channel 00/0 : 92[4000] -> 88[4000] [send] via NET/IB/0 comm 0x2af294000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO Connected all trees comm 0x2ab264000ab0 nRanks 96 busId 4000
e08r3n07:20445:21399 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n07:20445:21399 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n07:20447:21419 [1] NCCL INFO Connected all trees comm 0x2b8d08000ab0 nRanks 96 busId 26000
e08r3n07:20447:21419 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n07:20447:21419 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n06:24083:25278 [1] NCCL INFO Channel 00 : 73[26000] -> 72[4000] via P2P/IPC comm 0x2acfe4000ab0 nRanks 96
e08r3n06:24083:25278 [1] NCCL INFO Channel 01 : 73[26000] -> 72[4000] via P2P/IPC comm 0x2acfe4000ab0 nRanks 96
e08r3n06:24084:25277 [2] NCCL INFO Connected all trees comm 0x2b7908000ab0 nRanks 96 busId 43000
e08r3n06:24084:25277 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n06:24084:25277 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10508:11619 [0] NCCL INFO Connected all trees comm 0x2b625c000ab0 nRanks 96 busId 4000
e08r2n12:10508:11619 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10508:11619 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10509:11625 [1] NCCL INFO Connected all trees comm 0x2bacb8000ab0 nRanks 96 busId 26000
e08r2n12:10509:11625 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10509:11625 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n08:14727:15954 [0] NCCL INFO Channel 00/0 : 88[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac6a8000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 00/0 : 56[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af198000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Channel 00/0 : 28[4000] -> 24[4000] [send] via NET/IB/0 comm 0x2b227c000ab0 nRanks 96
e08r3n03:24229:25255 [0] NCCL INFO Channel 00/0 : 60[4000] -> 56[4000] [send] via NET/IB/0 comm 0x2ba43c000ab0 nRanks 96
e08r3n04:31686:32764 [0] NCCL INFO Channel 01/0 : 68[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b08c0000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Channel 01/0 : 72[4000] -> 69[26000] [send] via NET/IB/0 comm 0x2b280c000ab0 nRanks 96
e08r2n16:15463:16564 [0] NCCL INFO Channel 01/0 : 36[4000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af018000ab0 nRanks 96
e08r3n05:26315:27340 [0] NCCL INFO Channel 01/0 : 68[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b7410000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 01/0 : 44[4000] -> 36[4000] [send] via NET/IB/0 comm 0x2aff70000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 01/0 : 12[4000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6d8c000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO Connected all trees comm 0x2b227c000ab0 nRanks 96 busId 4000
e08r2n15:23877:25081 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23877:25081 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n15:23879:25082 [1] NCCL INFO Connected all trees comm 0x2b0918000ab0 nRanks 96 busId 26000
e08r2n15:23879:25082 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23879:25082 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26317:27342 [1] NCCL INFO Channel 00 : 69[26000] -> 68[4000] via P2P/IPC comm 0x2b7c88000ab0 nRanks 96
e08r3n05:26317:27342 [1] NCCL INFO Channel 01 : 69[26000] -> 68[4000] via P2P/IPC comm 0x2b7c88000ab0 nRanks 96
e08r3n06:24081:25273 [0] NCCL INFO Connected all trees comm 0x2b280c000ab0 nRanks 96 busId 4000
e08r3n06:24081:25273 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n06:24081:25273 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n06:24083:25278 [1] NCCL INFO Connected all trees comm 0x2acfe4000ab0 nRanks 96 busId 26000
e08r3n06:24083:25278 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n06:24083:25278 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26315:27340 [0] NCCL INFO Connected all trees comm 0x2b7410000ab0 nRanks 96 busId 4000
e08r3n05:26315:27340 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n05:26315:27340 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26317:27342 [1] NCCL INFO Connected all trees comm 0x2b7c88000ab0 nRanks 96 busId 26000
e08r3n05:26318:27350 [2] NCCL INFO Connected all trees comm 0x2b62cc000ab0 nRanks 96 busId 43000
e08r3n05:26317:27342 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n05:26317:27342 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26318:27350 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n05:26318:27350 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31686:32764 [0] NCCL INFO Connected all trees comm 0x2b08c0000ab0 nRanks 96 busId 4000
e08r3n04:31686:32764 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n04:31686:32764 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31688:304 [1] NCCL INFO Connected all trees comm 0x2af228000ab0 nRanks 96 busId 26000
e08r3n04:31688:304 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n04:31688:304 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n10:20256:21447 [0] NCCL INFO Channel 00/0 : 92[4000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b6488000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Channel 01/0 : 84[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac6a8000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Channel 01/0 : 52[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af198000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 00/0 : 60[4000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b57b8000ab0 nRanks 96
e08r2n14:20092:21169 [0] NCCL INFO Channel 01/0 : 24[4000] -> 21[26000] [send] via NET/IB/0 comm 0x2aec8c000ab0 nRanks 96
e08r2n19:10427:11596 [0] NCCL INFO Channel 00/0 : 44[4000] -> 40[4000] [send] via NET/IB/0 comm 0x2aff70000ab0 nRanks 96
e08r2n11:7501:8711 [0] NCCL INFO Channel 00/0 : 12[4000] -> 8[4000] [send] via NET/IB/0 comm 0x2b6d8c000ab0 nRanks 96
e08r3n07:20445:21399 [0] NCCL INFO comm 0x2ab264000ab0 rank 76 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n07:20448:21421 [2] NCCL INFO comm 0x2b4478000ab0 rank 78 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n07:20447:21419 [1] NCCL INFO comm 0x2b8d08000ab0 rank 77 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n07:20449:21403 [3] NCCL INFO comm 0x2b2570000ab0 rank 79 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n17:22820:23808 [0] NCCL INFO Channel 00/0 : 41[26000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b02c0000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Channel 00/0 : 9[26000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af210000ab0 nRanks 96
e08r2n12:10508:11619 [0] NCCL INFO comm 0x2b625c000ab0 rank 16 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n12:10509:11625 [1] NCCL INFO comm 0x2bacb8000ab0 rank 17 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n11:31583:32648 [0] NCCL INFO Connected all trees comm 0x2af294000ab0 nRanks 96 busId 4000
e08r3n11:31583:32648 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10510:11624 [2] NCCL INFO comm 0x2b5f60000ab0 rank 18 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n11:31583:32648 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n11:31585:32654 [1] NCCL INFO Connected all trees comm 0x2aeaa8000ab0 nRanks 96 busId 26000
e08r3n11:31585:32654 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n11:31585:32654 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10511:11620 [3] NCCL INFO comm 0x2b3e1c000ab0 rank 19 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n03:24229:25255 [0] NCCL INFO Connected all trees comm 0x2ba43c000ab0 nRanks 96 busId 4000
e08r3n03:24229:25255 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n03:24229:25255 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n03:24231:25246 [1] NCCL INFO Connected all trees comm 0x2b3040000ab0 nRanks 96 busId 26000
e08r2n19:10427:11596 [0] NCCL INFO Connected all trees comm 0x2aff70000ab0 nRanks 96 busId 4000
e08r3n03:24231:25246 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n03:24231:25246 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10427:11596 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n19:10427:11596 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20092:21169 [0] NCCL INFO Connected all trees comm 0x2aec8c000ab0 nRanks 96 busId 4000
e08r2n19:10428:11592 [1] NCCL INFO Connected all trees comm 0x2b3348000ab0 nRanks 96 busId 26000
e08r2n14:20092:21169 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n19:10428:11592 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n19:10428:11592 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20092:21169 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20094:21174 [1] NCCL INFO Connected all trees comm 0x2b550c000ab0 nRanks 96 busId 26000
e08r2n14:20094:21174 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n14:20094:21174 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7501:8711 [0] NCCL INFO Connected all trees comm 0x2b6d8c000ab0 nRanks 96 busId 4000
e08r2n11:7501:8711 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n11:7501:8711 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7502:8716 [1] NCCL INFO Connected all trees comm 0x2b7cdc000ab0 nRanks 96 busId 26000
e08r2n11:7502:8716 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n11:7502:8716 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26094:27113 [1] NCCL INFO Channel 00 : 21[26000] -> 20[4000] via P2P/IPC comm 0x2b2d08000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO Connected all trees comm 0x2ac6a8000ab0 nRanks 96 busId 4000
e08r2n13:26094:27113 [1] NCCL INFO Channel 01 : 21[26000] -> 20[4000] via P2P/IPC comm 0x2b2d08000ab0 nRanks 96
e08r3n08:14727:15954 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14727:15954 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n08:14729:15948 [1] NCCL INFO Connected all trees comm 0x2b1290000ab0 nRanks 96 busId 26000
e08r3n08:14729:15948 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14729:15948 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2342:3554 [1] NCCL INFO Channel 00 : 9[26000] -> 8[4000] via P2P/IPC comm 0x2ac06c000ab0 nRanks 96
e08r2n18:2636:3716 [1] NCCL INFO Channel 00 : 41[26000] -> 40[4000] via P2P/IPC comm 0x2ac578000ab0 nRanks 96
e08r2n10:2342:3554 [1] NCCL INFO Channel 01 : 9[26000] -> 8[4000] via P2P/IPC comm 0x2ac06c000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO Connected all trees comm 0x2af198000ab0 nRanks 96 busId 4000
e08r2n18:2636:3716 [1] NCCL INFO Channel 01 : 41[26000] -> 40[4000] via P2P/IPC comm 0x2ac578000ab0 nRanks 96
e08r3n00:29325:30324 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n00:29325:30324 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n00:29327:30330 [1] NCCL INFO Connected all trees comm 0x2b149c000ab0 nRanks 96 busId 26000
e08r3n00:29327:30330 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26093:27103 [0] NCCL INFO Connected all trees comm 0x2b19ac000ab0 nRanks 96 busId 4000
e08r3n00:29327:30330 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26093:27103 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26093:27103 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26094:27113 [1] NCCL INFO Connected all trees comm 0x2b2d08000ab0 nRanks 96 busId 26000
e08r2n13:26095:27111 [2] NCCL INFO Connected all trees comm 0x2b1d90000ab0 nRanks 96 busId 43000
e08r2n13:26094:27113 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26094:27113 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26095:27111 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26095:27111 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2343:3556 [2] NCCL INFO Connected all trees comm 0x2b14cc000ab0 nRanks 96 busId 43000
e08r2n10:2343:3556 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2637:3714 [2] NCCL INFO Connected all trees comm 0x2b36e4000ab0 nRanks 96 busId 43000
e08r2n18:2637:3714 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2343:3556 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n18:2637:3714 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n10:20256:21447 [0] NCCL INFO Channel 01/0 : 88[4000] -> 85[26000] [send] via NET/IB/0 comm 0x2b6488000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Channel 01/0 : 56[4000] -> 53[26000] [send] via NET/IB/0 comm 0x2b57b8000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Channel 01/0 : 40[4000] -> 37[26000] [send] via NET/IB/0 comm 0x2ac970000ab0 nRanks 96
e08r2n10:2340:3530 [0] NCCL INFO Channel 01/0 : 8[4000] -> 5[26000] [send] via NET/IB/0 comm 0x2b76f0000ab0 nRanks 96
e08r2n15:23880:25090 [2] NCCL INFO comm 0x2b1de8000ab0 rank 30 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n15:23879:25082 [1] NCCL INFO comm 0x2b0918000ab0 rank 29 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n17:22820:23808 [0] NCCL INFO Channel 01/0 : 36[4000] -> 32[4000] [send] via NET/IB/0 comm 0x2b02c0000ab0 nRanks 96
e08r2n15:23881:25092 [3] NCCL INFO comm 0x2b7d9c000ab0 rank 31 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n09:28728:29878 [0] NCCL INFO Channel 01/0 : 4[4000] -> 0[4000] [send] via NET/IB/0 comm 0x2af210000ab0 nRanks 96
e08r2n15:23877:25081 [0] NCCL INFO comm 0x2b227c000ab0 rank 28 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n06:24084:25277 [2] NCCL INFO comm 0x2b7908000ab0 rank 74 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n06:24085:25276 [3] NCCL INFO comm 0x2b6f04000ab0 rank 75 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n05:26317:27342 [1] NCCL INFO comm 0x2b7c88000ab0 rank 69 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n05:26315:27340 [0] NCCL INFO comm 0x2b7410000ab0 rank 68 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n05:26319:27347 [3] NCCL INFO comm 0x2ae1cc000ab0 rank 71 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n04:31689:307 [2] NCCL INFO comm 0x2ad454000ab0 rank 66 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n04:31690:302 [3] NCCL INFO comm 0x2ac5d0000ab0 rank 67 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n06:24081:25273 [0] NCCL INFO comm 0x2b280c000ab0 rank 72 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n06:24083:25278 [1] NCCL INFO comm 0x2acfe4000ab0 rank 73 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n05:26318:27350 [2] NCCL INFO comm 0x2b62cc000ab0 rank 70 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n04:31686:32764 [0] NCCL INFO comm 0x2b08c0000ab0 rank 64 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n04:31688:304 [1] NCCL INFO comm 0x2af228000ab0 rank 65 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n10:20256:21447 [0] NCCL INFO Connected all trees comm 0x2b6488000ab0 nRanks 96 busId 4000
e08r3n10:20256:21447 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n10:20256:21447 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n10:20258:21448 [1] NCCL INFO Connected all trees comm 0x2b129c000ab0 nRanks 96 busId 26000
e08r3n10:20258:21448 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n10:20258:21448 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10074:11130 [1] NCCL INFO Channel 00 : 85[26000] -> 84[4000] via P2P/IPC comm 0x2aed0c000ab0 nRanks 96
e08r3n09:10074:11130 [1] NCCL INFO Channel 01 : 85[26000] -> 84[4000] via P2P/IPC comm 0x2aed0c000ab0 nRanks 96
e08r3n02:14034:15017 [0] NCCL INFO Connected all trees comm 0x2b57b8000ab0 nRanks 96 busId 4000
e08r3n02:14034:15017 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n02:14034:15017 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14036:15025 [1] NCCL INFO Connected all trees comm 0x2b03ec000ab0 nRanks 96 busId 26000
e08r3n02:14036:15025 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n02:14036:15025 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10072:11133 [0] NCCL INFO Connected all trees comm 0x2ae258000ab0 nRanks 96 busId 4000
e08r3n09:10072:11133 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n09:10072:11133 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10074:11130 [1] NCCL INFO Connected all trees comm 0x2aed0c000ab0 nRanks 96 busId 26000
e08r3n09:10075:11136 [2] NCCL INFO Connected all trees comm 0x2b9dc4000ab0 nRanks 96 busId 43000
e08r3n09:10074:11130 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n09:10074:11130 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10075:11136 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n09:10075:11136 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n01:21694:22707 [1] NCCL INFO Channel 00 : 53[26000] -> 52[4000] via P2P/IPC comm 0x2b91cc000ab0 nRanks 96
e08r3n01:21694:22707 [1] NCCL INFO Channel 01 : 53[26000] -> 52[4000] via P2P/IPC comm 0x2b91cc000ab0 nRanks 96
e08r2n18:2634:3704 [0] NCCL INFO Connected all trees comm 0x2ac970000ab0 nRanks 96 busId 4000
e08r2n18:2634:3704 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2634:3704 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2340:3530 [0] NCCL INFO Connected all trees comm 0x2b76f0000ab0 nRanks 96 busId 4000
e08r2n18:2636:3716 [1] NCCL INFO Connected all trees comm 0x2ac578000ab0 nRanks 96 busId 26000
e08r2n10:2340:3530 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2340:3530 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n18:2636:3716 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2636:3716 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2342:3554 [1] NCCL INFO Connected all trees comm 0x2ac06c000ab0 nRanks 96 busId 26000
e08r2n10:2342:3554 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2342:3554 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n01:21692:22698 [0] NCCL INFO Connected all trees comm 0x2accfc000ab0 nRanks 96 busId 4000
e08r3n01:21692:22698 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n01:21692:22698 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n01:21694:22707 [1] NCCL INFO Connected all trees comm 0x2b91cc000ab0 nRanks 96 busId 26000
e08r3n01:21695:22709 [2] NCCL INFO Connected all trees comm 0x2ad544000ab0 nRanks 96 busId 43000
e08r3n01:21694:22707 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n09:28729:29883 [1] NCCL INFO Channel 00 : 5[26000] -> 4[4000] via P2P/IPC comm 0x2b6e70000ab0 nRanks 96
e08r3n01:21695:22709 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n01:21694:22707 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n01:21695:22709 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22822:23809 [1] NCCL INFO Channel 00 : 37[26000] -> 36[4000] via P2P/IPC comm 0x2b7c0c000ab0 nRanks 96
e08r2n09:28729:29883 [1] NCCL INFO Channel 01 : 5[26000] -> 4[4000] via P2P/IPC comm 0x2b6e70000ab0 nRanks 96
e08r2n17:22822:23809 [1] NCCL INFO Channel 01 : 37[26000] -> 36[4000] via P2P/IPC comm 0x2b7c0c000ab0 nRanks 96
e08r2n09:28728:29878 [0] NCCL INFO Connected all trees comm 0x2af210000ab0 nRanks 96 busId 4000
e08r2n09:28728:29878 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n09:28728:29878 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22820:23808 [0] NCCL INFO Connected all trees comm 0x2b02c0000ab0 nRanks 96 busId 4000
e08r2n17:22820:23808 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n17:22820:23808 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n09:28730:29888 [2] NCCL INFO Connected all trees comm 0x2b25f4000ab0 nRanks 96 busId 43000
e08r2n09:28729:29883 [1] NCCL INFO Connected all trees comm 0x2b6e70000ab0 nRanks 96 busId 26000
e08r2n09:28729:29883 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n09:28729:29883 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n09:28730:29888 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n09:28730:29888 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22822:23809 [1] NCCL INFO Connected all trees comm 0x2b7c0c000ab0 nRanks 96 busId 26000
e08r2n17:22822:23809 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n17:22822:23809 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22823:23812 [2] NCCL INFO Connected all trees comm 0x2af978000ab0 nRanks 96 busId 43000
e08r2n17:22823:23812 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n17:22823:23812 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15463:16564 [0] NCCL INFO Connected all trees comm 0x2af018000ab0 nRanks 96 busId 4000
e08r2n16:15463:16564 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n16:15463:16564 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15465:16568 [1] NCCL INFO Connected all trees comm 0x2ad1a8000ab0 nRanks 96 busId 26000
e08r2n16:15465:16568 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11393:12596 [0] NCCL INFO Connected all trees comm 0x2afee0000ab0 nRanks 96 busId 4000
e08r2n16:15465:16568 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n08:11393:12596 [0] NCCL INFO Using tuning table 0 with LL128 disabled
e08r2n08:11393:12596 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11393:12596 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n08:11395:12832 [1] NCCL INFO Connected all trees comm 0x2b218c000ab0 nRanks 96 busId 26000
e08r2n08:11395:12832 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11395:12832 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n11:31586:32658 [2] NCCL INFO comm 0x2b3058000ab0 rank 94 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n11:31583:32648 [0] NCCL INFO comm 0x2af294000ab0 rank 92 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n11:31587:32657 [3] NCCL INFO comm 0x2b8dd8000ab0 rank 95 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n11:31585:32654 [1] NCCL INFO comm 0x2aeaa8000ab0 rank 93 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n19:10429:11597 [2] NCCL INFO comm 0x2b019c000ab0 rank 46 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n19:10427:11596 [0] NCCL INFO comm 0x2aff70000ab0 rank 44 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n03:24232:25250 [2] NCCL INFO comm 0x2acd78000ab0 rank 62 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n11:7501:8711 [0] NCCL INFO comm 0x2b6d8c000ab0 rank 12 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n14:20095:21178 [2] NCCL INFO comm 0x2bac40000ab0 rank 26 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n19:10428:11592 [1] NCCL INFO comm 0x2b3348000ab0 rank 45 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n03:24233:25253 [3] NCCL INFO comm 0x2b66e0000ab0 rank 63 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n19:10430:11601 [3] NCCL INFO comm 0x2b6930000ab0 rank 47 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n11:7504:8721 [3] NCCL INFO comm 0x2baa1c000ab0 rank 15 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n03:24229:25255 [0] NCCL INFO comm 0x2ba43c000ab0 rank 60 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n08:14730:15934 [2] NCCL INFO comm 0x2aee98000ab0 rank 82 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n08:14727:15954 [0] NCCL INFO comm 0x2ac6a8000ab0 rank 80 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n08:14729:15948 [1] NCCL INFO comm 0x2b1290000ab0 rank 81 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n14:20094:21174 [1] NCCL INFO comm 0x2b550c000ab0 rank 25 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n00:29325:30324 [0] NCCL INFO comm 0x2af198000ab0 rank 48 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n00:29328:30332 [2] NCCL INFO comm 0x2ad580000ab0 rank 50 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n00:29330:30333 [3] NCCL INFO comm 0x2b0d6c000ab0 rank 51 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n13:26094:27113 [1] NCCL INFO comm 0x2b2d08000ab0 rank 21 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n13:26096:27117 [3] NCCL INFO comm 0x2b83c0000ab0 rank 23 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n14:20096:21173 [3] NCCL INFO comm 0x2b5e80000ab0 rank 27 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n03:24231:25246 [1] NCCL INFO comm 0x2b3040000ab0 rank 61 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n14:20092:21169 [0] NCCL INFO comm 0x2aec8c000ab0 rank 24 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n11:7502:8716 [1] NCCL INFO comm 0x2b7cdc000ab0 rank 13 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n08:14731:15932 [3] NCCL INFO comm 0x2b99d8000ab0 rank 83 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n13:26095:27111 [2] NCCL INFO comm 0x2b1d90000ab0 rank 22 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n11:7503:8719 [2] NCCL INFO comm 0x2b4fa4000ab0 rank 14 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n13:26093:27103 [0] NCCL INFO comm 0x2b19ac000ab0 rank 20 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n00:29327:30330 [1] NCCL INFO comm 0x2b149c000ab0 rank 49 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n10:20256:21447 [0] NCCL INFO comm 0x2b6488000ab0 rank 88 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n10:20259:21460 [2] NCCL INFO comm 0x2b837c000ab0 rank 90 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n10:20258:21448 [1] NCCL INFO comm 0x2b129c000ab0 rank 89 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n10:20260:21455 [3] NCCL INFO comm 0x2ace2c000ab0 rank 91 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n02:14037:15028 [2] NCCL INFO comm 0x2aecd0000ab0 rank 58 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n09:10076:11135 [3] NCCL INFO comm 0x2ab750000ab0 rank 87 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n09:10075:11136 [2] NCCL INFO comm 0x2b9dc4000ab0 rank 86 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n10:2340:3530 [0] NCCL INFO comm 0x2b76f0000ab0 rank 8 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n09:10072:11133 [0] NCCL INFO comm 0x2ae258000ab0 rank 84 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n18:2634:3704 [0] NCCL INFO comm 0x2ac970000ab0 rank 40 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n18:2637:3714 [2] NCCL INFO comm 0x2b36e4000ab0 rank 42 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n10:2342:3554 [1] NCCL INFO comm 0x2ac06c000ab0 rank 9 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n18:2636:3716 [1] NCCL INFO comm 0x2ac578000ab0 rank 41 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n01:21694:22707 [1] NCCL INFO comm 0x2b91cc000ab0 rank 53 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n18:2638:3717 [3] NCCL INFO comm 0x2ac998000ab0 rank 43 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n01:21695:22709 [2] NCCL INFO comm 0x2ad544000ab0 rank 54 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n02:14034:15017 [0] NCCL INFO comm 0x2b57b8000ab0 rank 56 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n09:28731:29886 [3] NCCL INFO comm 0x2b67e8000ab0 rank 7 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n09:28729:29883 [1] NCCL INFO comm 0x2b6e70000ab0 rank 5 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n17:22822:23809 [1] NCCL INFO comm 0x2b7c0c000ab0 rank 37 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n17:22824:23810 [3] NCCL INFO comm 0x2ba348000ab0 rank 39 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n17:22823:23812 [2] NCCL INFO comm 0x2af978000ab0 rank 38 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n17:22820:23808 [0] NCCL INFO comm 0x2b02c0000ab0 rank 36 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n08:11393:12596 [0] NCCL INFO comm 0x2afee0000ab0 rank 0 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n16:15467:16571 [3] NCCL INFO comm 0x2b1dc0000ab0 rank 35 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n08:11395:12832 [1] NCCL INFO comm 0x2b218c000ab0 rank 1 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n16:15463:16564 [0] NCCL INFO comm 0x2af018000ab0 rank 32 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n01:21696:22700 [3] NCCL INFO comm 0x2b9c1c000ab0 rank 55 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n02:14036:15025 [1] NCCL INFO comm 0x2b03ec000ab0 rank 57 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n02:14038:15023 [3] NCCL INFO comm 0x2b2d40000ab0 rank 59 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n09:10074:11130 [1] NCCL INFO comm 0x2aed0c000ab0 rank 85 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n10:2343:3556 [2] NCCL INFO comm 0x2b14cc000ab0 rank 10 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n10:2344:3537 [3] NCCL INFO comm 0x2b4a18000ab0 rank 11 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n01:21692:22698 [0] NCCL INFO comm 0x2accfc000ab0 rank 52 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n08:11396:12836 [2] NCCL INFO comm 0x2b56b4000ab0 rank 2 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n16:15466:16578 [2] NCCL INFO comm 0x2bafec000ab0 rank 34 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n16:15465:16568 [1] NCCL INFO comm 0x2ad1a8000ab0 rank 33 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n08:11397:12837 [3] NCCL INFO comm 0x2b6a18000ab0 rank 3 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n09:28728:29878 [0] NCCL INFO comm 0x2af210000ab0 rank 4 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n09:28730:29888 [2] NCCL INFO comm 0x2b25f4000ab0 rank 6 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
[2023-10-27 11:25:52,599] [INFO] [partition_parameters.py:454:__exit__] finished initializing model with 7.51B parameters
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题

input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题

label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题

input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
input: <reserved_106>写一篇关于气候变化对海洋生态的影响的文章。
<reserved_107>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
label: </s>好的，以下是你要求的文章：
气候变化对海洋生态的影响
气候变化对海洋生态产生了广泛的影响。全球变暖导致海洋温度上升，这可能会对许多水生生物的生存、繁殖和迁移造成巨大的影响。另外，海洋酸化也是一个问题
Parameter Offload: Total persistent parameters: 266240 in 65 params
e08r2n08:11393:19499 [0] NCCL INFO Using network IB
e08r2n17:22824:25586 [3] NCCL INFO Using network IB
e08r2n12:10511:13649 [3] NCCL INFO Using network IB
e08r2n17:22823:25587 [2] NCCL INFO Using network IB
e08r2n12:10510:13650 [2] NCCL INFO Using network IB
e08r3n03:24232:27074 [2] NCCL INFO Using network IB
e08r2n13:26095:29022 [2] NCCL INFO Using network IB
e08r2n08:11397:19501 [3] NCCL INFO Using network IB
e08r2n13:26096:29023 [3] NCCL INFO Using network IB
e08r2n11:7503:10713 [2] NCCL INFO Using network IB
e08r3n11:31587:2055 [3] NCCL INFO Using network IB
e08r3n07:20448:22834 [2] NCCL INFO Using network IB
e08r2n10:2343:5283 [2] NCCL INFO Using network IB
e08r3n05:26319:29370 [3] NCCL INFO Using network IB
e08r2n08:11396:19502 [2] NCCL INFO Using network IB
e08r2n11:7504:10715 [3] NCCL INFO Using network IB
e08r3n11:31586:2056 [2] NCCL INFO Using network IB
e08r3n07:20449:22835 [3] NCCL INFO Using network IB
e08r2n10:2344:5284 [3] NCCL INFO Using network IB
e08r3n05:26318:29369 [2] NCCL INFO Using network IB
e08r3n10:20260:23238 [3] NCCL INFO Using network IB
e08r2n18:2638:5671 [3] NCCL INFO Using network IB
e08r2n15:23880:26728 [2] NCCL INFO Using network IB
e08r3n06:24084:27037 [2] NCCL INFO Using network IB
e08r2n14:20095:23164 [2] NCCL INFO Using network IB
e08r3n09:10075:13072 [2] NCCL INFO Using network IB
e08r2n15:23881:26730 [3] NCCL INFO Using network IB
e08r3n06:24085:27036 [3] NCCL INFO Using network IB
e08r2n14:20096:23166 [3] NCCL INFO Using network IB
e08r3n09:10076:13074 [3] NCCL INFO Using network IB
e08r3n04:31690:1945 [3] NCCL INFO Using network IB
e08r2n16:15467:18182 [3] NCCL INFO Using network IB
e08r3n01:21694:24038 [1] NCCL INFO Using network IB
e08r3n03:24233:27076 [3] NCCL INFO Using network IB
e08r3n00:29330:31833 [3] NCCL INFO Using network IB
e08r3n08:14727:17764 [0] NCCL INFO Using network IB
e08r3n02:14038:16484 [3] NCCL INFO Using network IB
e08r2n08:11395:19506 [1] NCCL INFO Using network IB
e08r2n15:23877:26732 [0] NCCL INFO Using network IB
e08r2n14:20092:23168 [0] NCCL INFO Using network IB
e08r3n09:10072:13076 [0] NCCL INFO Using network IB
e08r3n04:31689:1947 [2] NCCL INFO Using network IB
e08r2n16:15466:18183 [2] NCCL INFO Using network IB
e08r3n01:21692:24039 [0] NCCL INFO Using network IB
e08r3n03:24229:27078 [0] NCCL INFO Using network IB
e08r3n08:14730:17765 [2] NCCL INFO Using network IB
e08r3n02:14037:16485 [2] NCCL INFO Using network IB
e08r3n07:20445:22838 [0] NCCL INFO Using network IB
e08r3n10:20259:23240 [2] NCCL INFO Using network IB
e08r3n11:31583:2060 [0] NCCL INFO Using network IB
e08r2n13:26093:29026 [0] NCCL INFO Using network IB
e08r2n19:10429:13634 [2] NCCL INFO Using network IB
e08r2n12:10508:13653 [0] NCCL INFO Using network IB
e08r3n05:26315:29373 [0] NCCL INFO Using network IB
e08r2n09:28729:31686 [1] NCCL INFO Using network IB
e08r2n11:7501:10717 [0] NCCL INFO Using network IB
e08r2n17:22822:25590 [1] NCCL INFO Using network IB
e08r2n10:2340:5287 [0] NCCL INFO Using network IB
e08r2n15:23879:26734 [1] NCCL INFO Using network IB
e08r2n14:20094:23169 [1] NCCL INFO Using network IB
e08r3n09:10074:13077 [1] NCCL INFO Using network IB
e08r3n04:31686:1949 [0] NCCL INFO Using network IB
e08r2n16:15465:18186 [1] NCCL INFO Using network IB
e08r3n01:21696:24040 [3] NCCL INFO Using network IB
e08r3n03:24231:27079 [1] NCCL INFO Using network IB
e08r3n08:14731:17766 [3] NCCL INFO Using network IB
e08r3n02:14036:16488 [1] NCCL INFO Using network IB
e08r3n07:20447:22839 [1] NCCL INFO Using network IB
e08r3n10:20256:23242 [0] NCCL INFO Using network IB
e08r3n11:31585:2059 [1] NCCL INFO Using network IB
e08r2n13:26094:29027 [1] NCCL INFO Using network IB
e08r2n19:10430:13635 [3] NCCL INFO Using network IB
e08r2n12:10509:13654 [1] NCCL INFO Using network IB
e08r3n05:26317:29374 [1] NCCL INFO Using network IB
e08r2n09:28730:31687 [2] NCCL INFO Using network IB
e08r2n11:7502:10718 [1] NCCL INFO Using network IB
e08r2n17:22820:25591 [0] NCCL INFO Using network IB
e08r2n10:2342:5288 [1] NCCL INFO Using network IB
e08r2n18:2637:5674 [2] NCCL INFO Using network IB
e08r3n00:29325:31835 [0] NCCL INFO Using network IB
e08r3n04:31688:1950 [1] NCCL INFO Using network IB
e08r2n16:15463:18187 [0] NCCL INFO Using network IB
e08r3n01:21695:24041 [2] NCCL INFO Using network IB
e08r3n08:14729:17767 [1] NCCL INFO Using network IB
e08r3n02:14034:16489 [0] NCCL INFO Using network IB
e08r3n10:20258:23243 [1] NCCL INFO Using network IB
e08r2n19:10428:13638 [1] NCCL INFO Using network IB
e08r2n09:28731:31688 [3] NCCL INFO Using network IB
e08r2n18:2636:5675 [1] NCCL INFO Using network IB
e08r3n00:29327:31836 [1] NCCL INFO Using network IB
e08r3n06:24083:27040 [1] NCCL INFO Using network IB
e08r2n19:10427:13639 [0] NCCL INFO Using network IB
e08r2n09:28728:31689 [0] NCCL INFO Using network IB
e08r2n18:2634:5676 [0] NCCL INFO Using network IB
e08r3n00:29328:31837 [2] NCCL INFO Using network IB
e08r3n06:24081:27041 [0] NCCL INFO Using network IB
e08r2n11:7504:10715 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n08:11393:19499 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n17:22822:25590 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n10:20259:23240 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n09:10075:13072 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n14:20096:23166 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n14:20092:23168 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n15:23880:26728 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n15:23881:26730 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n13:26095:29022 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n13:26093:29026 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n14:20094:23169 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n11:7503:10713 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n10:20260:23238 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n11:7502:10718 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n13:26094:29027 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n08:11397:19501 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n17:22823:25587 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n17:22820:25591 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n11:7501:10717 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n15:23877:26732 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n13:26096:29023 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n15:23879:26734 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n08:11395:19506 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n12:10508:13653 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n08:11396:19502 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n05:26315:29373 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n04:31690:1945 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n09:28731:31688 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n09:28728:31689 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n04:31689:1947 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n05:26317:29374 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n01:21692:24039 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n10:20256:23242 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n06:24083:27040 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n12:10509:13654 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n10:20258:23243 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n19:10430:13635 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n03:24229:27078 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n00:29325:31835 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n19:10429:13634 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n17:22824:25586 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n12:10510:13650 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n02:14034:16489 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n19:10427:13639 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n12:10511:13649 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n03:24233:27076 [3] NCCL INFO Setting affinity for GPU 0 to ff000000
e08r2n18:2638:5671 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n14:20095:23164 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n04:31688:1950 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n00:29327:31836 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n03:24232:27074 [2] NCCL INFO Setting affinity for GPU 0 to ff0000
e08r2n18:2634:5676 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n04:31686:1949 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n10:2342:5288 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n05:26319:29370 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n11:31586:2056 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n09:28729:31686 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n09:10076:13074 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n07:20447:22839 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n03:24231:27079 [1] NCCL INFO Setting affinity for GPU 0 to ff00
e08r2n18:2636:5675 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n10:2344:5284 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n05:26318:29369 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n11:31583:2060 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n09:28730:31687 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n02:14036:16488 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n00:29330:31833 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n06:24081:27041 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n08:14729:17767 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n18:2637:5674 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n10:2340:5287 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n11:31585:2059 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n00:29328:31837 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n01:21696:24040 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n19:10428:13638 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n10:2343:5283 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n11:31587:2055 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n01:21694:24038 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n06:24085:27036 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n09:10072:13076 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n08:14731:17766 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r2n16:15463:18187 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n02:14038:16484 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n01:21695:24041 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n06:24084:27037 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n09:10074:13077 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r2n16:15465:18186 [1] NCCL INFO Setting affinity for GPU 1 to ff00
e08r3n02:14037:16485 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n07:20448:22834 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n08:14727:17764 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r3n07:20449:22835 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n08:14730:17765 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r2n16:15467:18182 [3] NCCL INFO Setting affinity for GPU 3 to ff000000
e08r3n07:20445:22838 [0] NCCL INFO Setting affinity for GPU 0 to ff
e08r2n16:15466:18183 [2] NCCL INFO Setting affinity for GPU 2 to ff0000
e08r3n00:29330:31833 [3] NCCL INFO Trees [0] -1/-1/-1->51->50 [1] -1/-1/-1->51->50 comm 0x2b0fd8000ab0 nRanks 96 busId 63000
e08r3n00:29328:31837 [2] NCCL INFO Trees [0] 51/-1/-1->50->49 [1] 51/-1/-1->50->49 comm 0x2ad7f0000ab0 nRanks 96 busId 43000
e08r3n01:21692:24039 [0] NCCL INFO Trees [0] 53/-1/-1->52->57 [1] 53/48/-1->52->45 comm 0x2acf70000ab0 nRanks 96 busId 4000
e08r3n00:29327:31836 [1] NCCL INFO Trees [0] 50/40/-1->49->48 [1] 50/-1/-1->49->48 comm 0x2b1704000ab0 nRanks 96 busId 26000
e08r3n01:21694:24038 [1] NCCL INFO Trees [0] 54/-1/-1->53->52 [1] 54/56/-1->53->52 comm 0x2b9438000ab0 nRanks 96 busId 26000
e08r3n02:14034:16489 [0] NCCL INFO Trees [0] 57/60/-1->56->48 [1] 57/-1/-1->56->53 comm 0x2b5a20000ab0 nRanks 96 busId 4000
e08r3n00:29325:31835 [0] NCCL INFO Trees [0] 49/56/-1->48->32 [1] 49/-1/-1->48->52 comm 0x2af400000ab0 nRanks 96 busId 4000
e08r2n19:10430:13635 [3] NCCL INFO Trees [0] -1/-1/-1->47->46 [1] -1/-1/-1->47->46 comm 0x2b6b98000ab0 nRanks 96 busId 63000
e08r3n01:21695:24041 [2] NCCL INFO Trees [0] 55/-1/-1->54->53 [1] 55/-1/-1->54->53 comm 0x2ad7ac000ab0 nRanks 96 busId 43000
e08r2n19:10429:13634 [2] NCCL INFO Trees [0] 47/-1/-1->46->45 [1] 47/-1/-1->46->45 comm 0x2b0404000ab0 nRanks 96 busId 43000
e08r3n01:21696:24040 [3] NCCL INFO Trees [0] -1/-1/-1->55->54 [1] -1/-1/-1->55->54 comm 0x2b9e90000ab0 nRanks 96 busId 63000
e08r3n04:31690:1945 [3] NCCL INFO Trees [0] -1/-1/-1->67->66 [1] -1/-1/-1->67->66 comm 0x2ac844000ab0 nRanks 96 busId 63000
e08r3n02:14036:16488 [1] NCCL INFO Trees [0] 58/52/-1->57->56 [1] 58/-1/-1->57->56 comm 0x2b0654000ab0 nRanks 96 busId 26000
e08r3n04:31688:1950 [1] NCCL INFO Trees [0] 66/32/-1->65->64 [1] 66/-1/-1->65->64 comm 0x2af494000ab0 nRanks 96 busId 26000
e08r3n02:14037:16485 [2] NCCL INFO Trees [0] 59/-1/-1->58->57 [1] 59/-1/-1->58->57 comm 0x2aef3c000ab0 nRanks 96 busId 43000
e08r3n03:24231:27079 [1] NCCL INFO Trees [0] 62/-1/-1->61->60 [1] 62/76/-1->61->60 comm 0x2b32ac000ab0 nRanks 96 busId 26000
e08r3n04:31689:1947 [2] NCCL INFO Trees [0] 67/-1/-1->66->65 [1] 67/-1/-1->66->65 comm 0x2ad6c0000ab0 nRanks 96 busId 43000
e08r3n03:24232:27074 [2] NCCL INFO Trees [0] 63/-1/-1->62->61 [1] 63/-1/-1->62->61 comm 0x2acfe4000ab0 nRanks 96 busId 43000
e08r3n02:14038:16484 [3] NCCL INFO Trees [0] -1/-1/-1->59->58 [1] -1/-1/-1->59->58 comm 0x2b2fa8000ab0 nRanks 96 busId 63000
e08r3n04:31686:1949 [0] NCCL INFO Trees [0] 65/80/-1->64->0 [1] 65/-1/-1->64->68 comm 0x2b0b28000ab0 nRanks 96 busId 4000
e08r2n17:22823:25587 [2] NCCL INFO Trees [0] 39/-1/-1->38->37 [1] 39/-1/-1->38->37 comm 0x2afbe4000ab0 nRanks 96 busId 43000
e08r2n17:22824:25586 [3] NCCL INFO Trees [0] -1/-1/-1->39->38 [1] -1/-1/-1->39->38 comm 0x2ba5b0000ab0 nRanks 96 busId 63000
e08r2n16:15465:18186 [1] NCCL INFO Trees [0] 34/16/-1->33->32 [1] 34/-1/-1->33->32 comm 0x2ad418000ab0 nRanks 96 busId 26000
e08r3n03:24233:27076 [3] NCCL INFO Trees [0] -1/-1/-1->63->62 [1] -1/-1/-1->63->62 comm 0x2b694c000ab0 nRanks 96 busId 63000
e08r2n17:22820:25591 [0] NCCL INFO Trees [0] 37/-1/-1->36->41 [1] 37/32/-1->36->44 comm 0x2b0528000ab0 nRanks 96 busId 4000
e08r3n03:24229:27078 [0] NCCL INFO Trees [0] 61/-1/-1->60->56 [1] 61/44/-1->60->29 comm 0x2ba6a8000ab0 nRanks 96 busId 4000
e08r2n15:23879:26734 [1] NCCL INFO Trees [0] 30/-1/-1->29->28 [1] 30/60/-1->29->28 comm 0x2b0b84000ab0 nRanks 96 busId 26000
e08r2n17:22822:25590 [1] NCCL INFO Trees [0] 38/-1/-1->37->36 [1] 38/40/-1->37->36 comm 0x2b7e78000ab0 nRanks 96 busId 26000
e08r2n15:23881:26730 [3] NCCL INFO Trees [0] -1/-1/-1->31->30 [1] -1/-1/-1->31->30 comm 0x2b8004000ab0 nRanks 96 busId 63000
e08r3n06:24083:27040 [1] NCCL INFO Trees [0] 74/68/-1->73->72 [1] 74/-1/-1->73->72 comm 0x2ad250000ab0 nRanks 96 busId 26000
e08r2n16:15467:18182 [3] NCCL INFO Trees [0] -1/-1/-1->35->34 [1] -1/-1/-1->35->34 comm 0x2b2028000ab0 nRanks 96 busId 63000
e08r3n05:26315:29373 [0] NCCL INFO Trees [0] 69/-1/-1->68->73 [1] 69/64/-1->68->76 comm 0x2b7678000ab0 nRanks 96 busId 4000
e08r2n15:23880:26728 [2] NCCL INFO Trees [0] 31/-1/-1->30->29 [1] 31/-1/-1->30->29 comm 0x2b2058000ab0 nRanks 96 busId 43000
e08r3n06:24084:27037 [2] NCCL INFO Trees [0] 75/-1/-1->74->73 [1] 75/-1/-1->74->73 comm 0x2b7b74000ab0 nRanks 96 busId 43000
e08r2n16:15466:18183 [2] NCCL INFO Trees [0] 35/-1/-1->34->33 [1] 35/-1/-1->34->33 comm 0x2bb254000ab0 nRanks 96 busId 43000
e08r3n05:26317:29374 [1] NCCL INFO Trees [0] 70/-1/-1->69->68 [1] 70/72/-1->69->68 comm 0x2b7ef4000ab0 nRanks 96 busId 26000
e08r3n10:20260:23238 [3] NCCL INFO Trees [0] -1/-1/-1->91->90 [1] -1/-1/-1->91->90 comm 0x2ad094000ab0 nRanks 96 busId 63000
e08r2n18:2634:5676 [0] NCCL INFO Trees [0] 41/44/-1->40->49 [1] 41/-1/-1->40->37 comm 0x2acbdc000ab0 nRanks 96 busId 4000
e08r2n19:10428:13638 [1] NCCL INFO Trees [0] 46/-1/-1->45->44 [1] 46/52/-1->45->44 comm 0x2b35b4000ab0 nRanks 96 busId 26000
e08r3n07:20447:22839 [1] NCCL INFO Trees [0] 78/-1/-1->77->76 [1] 78/84/-1->77->76 comm 0x2b8f74000ab0 nRanks 96 busId 26000
e08r3n06:24081:27041 [0] NCCL INFO Trees [0] 73/76/-1->72->81 [1] 73/-1/-1->72->69 comm 0x2b2a74000ab0 nRanks 96 busId 4000
e08r2n16:15463:18187 [0] NCCL INFO Trees [0] 33/48/-1->32->65 [1] 33/-1/-1->32->36 comm 0x2af288000ab0 nRanks 96 busId 4000
e08r3n05:26318:29369 [2] NCCL INFO Trees [0] 71/-1/-1->70->69 [1] 71/-1/-1->70->69 comm 0x2b6538000ab0 nRanks 96 busId 43000
e08r3n10:20258:23243 [1] NCCL INFO Trees [0] 90/84/-1->89->88 [1] 90/-1/-1->89->88 comm 0x2b150c000ab0 nRanks 96 busId 26000
e08r2n18:2636:5675 [1] NCCL INFO Trees [0] 42/36/-1->41->40 [1] 42/-1/-1->41->40 comm 0x2ac7e4000ab0 nRanks 96 busId 26000
e08r2n19:10427:13639 [0] NCCL INFO Trees [0] 45/-1/-1->44->40 [1] 45/36/-1->44->60 comm 0x2b01e0000ab0 nRanks 96 busId 4000
e08r3n07:20445:22838 [0] NCCL INFO Trees [0] 77/-1/-1->76->72 [1] 77/68/-1->76->61 comm 0x2ab4d0000ab0 nRanks 96 busId 4000
e08r3n09:10072:13076 [0] NCCL INFO Trees [0] 85/-1/-1->84->89 [1] 85/80/-1->84->77 comm 0x2ae4c4000ab0 nRanks 96 busId 4000
e08r3n11:31583:2060 [0] NCCL INFO Trees [0] 93/-1/-1->92->88 [1] 93/28/-1->92->-1 comm 0x2af4fc000ab0 nRanks 96 busId 4000
e08r3n08:14730:17765 [2] NCCL INFO Trees [0] 83/-1/-1->82->81 [1] 83/-1/-1->82->81 comm 0x2af108000ab0 nRanks 96 busId 43000
e08r2n08:11395:19506 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 comm 0x2b23f8000ab0 nRanks 96 busId 26000
e08r2n08:11393:19499 [0] NCCL INFO Channel 00/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
e08r2n08:11393:19499 [0] NCCL INFO Channel 01/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
e08r2n08:11393:19499 [0] NCCL INFO Trees [0] 1/64/-1->0->-1 [1] 1/-1/-1->0->4 comm 0x2b0148000ab0 nRanks 96 busId 4000
e08r2n08:11396:19502 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 comm 0x2b5920000ab0 nRanks 96 busId 43000
e08r3n06:24085:27036 [3] NCCL INFO Trees [0] -1/-1/-1->75->74 [1] -1/-1/-1->75->74 comm 0x2b716c000ab0 nRanks 96 busId 63000
e08r3n05:26319:29370 [3] NCCL INFO Trees [0] -1/-1/-1->71->70 [1] -1/-1/-1->71->70 comm 0x2ae434000ab0 nRanks 96 busId 63000
e08r3n10:20259:23240 [2] NCCL INFO Trees [0] 91/-1/-1->90->89 [1] 91/-1/-1->90->89 comm 0x2b85ec000ab0 nRanks 96 busId 43000
e08r2n18:2637:5674 [2] NCCL INFO Trees [0] 43/-1/-1->42->41 [1] 43/-1/-1->42->41 comm 0x2b3950000ab0 nRanks 96 busId 43000
e08r3n07:20448:22834 [2] NCCL INFO Trees [0] 79/-1/-1->78->77 [1] 79/-1/-1->78->77 comm 0x2b46e4000ab0 nRanks 96 busId 43000
e08r3n09:10074:13077 [1] NCCL INFO Trees [0] 86/-1/-1->85->84 [1] 86/88/-1->85->84 comm 0x2aef78000ab0 nRanks 96 busId 26000
e08r3n11:31585:2059 [1] NCCL INFO Trees [0] 94/-1/-1->93->92 [1] 94/-1/-1->93->92 comm 0x2aed10000ab0 nRanks 96 busId 26000
e08r3n08:14727:17764 [0] NCCL INFO Trees [0] 81/88/-1->80->64 [1] 81/-1/-1->80->84 comm 0x2ac910000ab0 nRanks 96 busId 4000
e08r2n10:2344:5284 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] -1/-1/-1->11->10 comm 0x2b4c80000ab0 nRanks 96 busId 63000
e08r2n08:11397:19501 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 comm 0x2b6c88000ab0 nRanks 96 busId 63000
e08r3n10:20256:23242 [0] NCCL INFO Trees [0] 89/92/-1->88->80 [1] 89/-1/-1->88->85 comm 0x2b66fc000ab0 nRanks 96 busId 4000
e08r2n18:2638:5671 [3] NCCL INFO Trees [0] -1/-1/-1->43->42 [1] -1/-1/-1->43->42 comm 0x2acc04000ab0 nRanks 96 busId 63000
e08r3n07:20449:22835 [3] NCCL INFO Trees [0] -1/-1/-1->79->78 [1] -1/-1/-1->79->78 comm 0x2b27d8000ab0 nRanks 96 busId 63000
e08r3n09:10076:13074 [3] NCCL INFO Trees [0] -1/-1/-1->87->86 [1] -1/-1/-1->87->86 comm 0x2ab9b8000ab0 nRanks 96 busId 63000
e08r3n11:31587:2055 [3] NCCL INFO Trees [0] -1/-1/-1->95->94 [1] -1/-1/-1->95->94 comm 0x2b9048000ab0 nRanks 96 busId 63000
e08r3n08:14729:17767 [1] NCCL INFO Trees [0] 82/72/-1->81->80 [1] 82/-1/-1->81->80 comm 0x2b1500000ab0 nRanks 96 busId 26000
e08r2n10:2343:5283 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 comm 0x2b1738000ab0 nRanks 96 busId 43000
e08r2n11:7501:10717 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] 13/4/-1->12->28 comm 0x2b6ff4000ab0 nRanks 96 busId 4000
e08r2n09:28729:31686 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/8/-1->5->4 comm 0x2b70dc000ab0 nRanks 96 busId 26000
e08r2n12:10509:13654 [1] NCCL INFO Trees [0] 18/8/-1->17->16 [1] 18/-1/-1->17->16 comm 0x2baf28000ab0 nRanks 96 busId 26000
e08r3n09:10075:13072 [2] NCCL INFO Trees [0] 87/-1/-1->86->85 [1] 87/-1/-1->86->85 comm 0x2ba030000ab0 nRanks 96 busId 43000
e08r3n11:31586:2056 [2] NCCL INFO Trees [0] 95/-1/-1->94->93 [1] 95/-1/-1->94->93 comm 0x2b32c4000ab0 nRanks 96 busId 43000
e08r3n08:14731:17766 [3] NCCL INFO Trees [0] -1/-1/-1->83->82 [1] -1/-1/-1->83->82 comm 0x2b9c40000ab0 nRanks 96 busId 63000
e08r2n10:2342:5288 [1] NCCL INFO Trees [0] 10/4/-1->9->8 [1] 10/-1/-1->9->8 comm 0x2ac2d8000ab0 nRanks 96 busId 26000
e08r2n11:7502:10718 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/20/-1->13->12 comm 0x2b7f44000ab0 nRanks 96 busId 26000
e08r2n09:28731:31688 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 comm 0x2b6a54000ab0 nRanks 96 busId 63000
e08r2n12:10510:13650 [2] NCCL INFO Trees [0] 19/-1/-1->18->17 [1] 19/-1/-1->18->17 comm 0x2b61d0000ab0 nRanks 96 busId 43000
e08r2n14:20094:23169 [1] NCCL INFO Trees [0] 26/20/-1->25->24 [1] 26/-1/-1->25->24 comm 0x2b5774000ab0 nRanks 96 busId 26000
e08r2n13:26094:29027 [1] NCCL INFO Trees [0] 22/-1/-1->21->20 [1] 22/24/-1->21->20 comm 0x2b2f78000ab0 nRanks 96 busId 26000
e08r2n10:2340:5287 [0] NCCL INFO Trees [0] 9/12/-1->8->17 [1] 9/-1/-1->8->5 comm 0x2b795c000ab0 nRanks 96 busId 4000
e08r2n11:7503:10713 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 comm 0x2b520c000ab0 nRanks 96 busId 43000
e08r2n09:28730:31687 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 comm 0x2b2864000ab0 nRanks 96 busId 43000
e08r2n12:10511:13649 [3] NCCL INFO Trees [0] -1/-1/-1->19->18 [1] -1/-1/-1->19->18 comm 0x2b408c000ab0 nRanks 96 busId 63000
e08r2n14:20092:23168 [0] NCCL INFO Trees [0] 25/28/-1->24->16 [1] 25/-1/-1->24->21 comm 0x2aeefc000ab0 nRanks 96 busId 4000
e08r2n13:26096:29023 [3] NCCL INFO Trees [0] -1/-1/-1->23->22 [1] -1/-1/-1->23->22 comm 0x2b8628000ab0 nRanks 96 busId 63000
e08r2n15:23877:26732 [0] NCCL INFO Trees [0] 29/-1/-1->28->24 [1] 29/12/-1->28->92 comm 0x2b24e8000ab0 nRanks 96 busId 4000
e08r2n11:7504:10715 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] -1/-1/-1->15->14 comm 0x2bac84000ab0 nRanks 96 busId 63000
e08r2n09:28728:31689 [0] NCCL INFO Trees [0] 5/-1/-1->4->9 [1] 5/0/-1->4->12 comm 0x2af480000ab0 nRanks 96 busId 4000
e08r2n12:10508:13653 [0] NCCL INFO Trees [0] 17/24/-1->16->33 [1] 17/-1/-1->16->20 comm 0x2b64c4000ab0 nRanks 96 busId 4000
e08r2n14:20095:23164 [2] NCCL INFO Trees [0] 27/-1/-1->26->25 [1] 27/-1/-1->26->25 comm 0x2baeac000ab0 nRanks 96 busId 43000
e08r2n13:26093:29026 [0] NCCL INFO Trees [0] 21/-1/-1->20->25 [1] 21/16/-1->20->13 comm 0x2b1c18000ab0 nRanks 96 busId 4000
e08r2n14:20096:23166 [3] NCCL INFO Trees [0] -1/-1/-1->27->26 [1] -1/-1/-1->27->26 comm 0x2b60ec000ab0 nRanks 96 busId 63000
e08r2n13:26095:29022 [2] NCCL INFO Trees [0] 23/-1/-1->22->21 [1] 23/-1/-1->22->21 comm 0x2b1ff8000ab0 nRanks 96 busId 43000
e08r3n01:21692:24039 [0] NCCL INFO Channel 00/0 : 51[63000] -> 52[4000] [receive] via NET/IB/0 comm 0x2acf70000ab0 nRanks 96
e08r3n00:29328:31837 [2] NCCL INFO Channel 00 : 50[43000] -> 51[63000] via P2P/IPC comm 0x2ad7f0000ab0 nRanks 96
e08r3n02:14034:16489 [0] NCCL INFO Channel 00/0 : 55[63000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b5a20000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 00/0 : 47[63000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af400000ab0 nRanks 96
e08r3n00:29327:31836 [1] NCCL INFO Channel 00 : 49[26000] -> 50[43000] via P2P/IPC comm 0x2b1704000ab0 nRanks 96
e08r3n00:29328:31837 [2] NCCL INFO Channel 01 : 50[43000] -> 51[63000] via P2P/IPC comm 0x2ad7f0000ab0 nRanks 96
e08r2n19:10429:13634 [2] NCCL INFO Channel 00 : 46[43000] -> 47[63000] via P2P/IPC comm 0x2b0404000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Channel 00/0 : 63[63000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b0b28000ab0 nRanks 96
e08r3n01:21694:24038 [1] NCCL INFO Channel 00 : 53[26000] -> 54[43000] via P2P/IPC comm 0x2b9438000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 00/0 : 59[63000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba6a8000ab0 nRanks 96
e08r3n01:21695:24041 [2] NCCL INFO Channel 00 : 54[43000] -> 55[63000] via P2P/IPC comm 0x2ad7ac000ab0 nRanks 96
e08r2n15:23879:26734 [1] NCCL INFO Channel 00 : 29[26000] -> 30[43000] via P2P/IPC comm 0x2b0b84000ab0 nRanks 96
e08r3n04:31688:1950 [1] NCCL INFO Channel 00 : 65[26000] -> 66[43000] via P2P/IPC comm 0x2af494000ab0 nRanks 96
e08r3n00:29327:31836 [1] NCCL INFO Channel 01 : 49[26000] -> 50[43000] via P2P/IPC comm 0x2b1704000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 00/0 : 35[63000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b0528000ab0 nRanks 96
e08r3n03:24231:27079 [1] NCCL INFO Channel 00 : 61[26000] -> 62[43000] via P2P/IPC comm 0x2b32ac000ab0 nRanks 96
e08r2n19:10429:13634 [2] NCCL INFO Channel 01 : 46[43000] -> 47[63000] via P2P/IPC comm 0x2b0404000ab0 nRanks 96
e08r3n04:31689:1947 [2] NCCL INFO Channel 00 : 66[43000] -> 67[63000] via P2P/IPC comm 0x2ad6c0000ab0 nRanks 96
e08r3n01:21694:24038 [1] NCCL INFO Channel 01 : 53[26000] -> 54[43000] via P2P/IPC comm 0x2b9438000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 00/0 : 31[63000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af288000ab0 nRanks 96
e08r3n03:24232:27074 [2] NCCL INFO Channel 00 : 62[43000] -> 63[63000] via P2P/IPC comm 0x2acfe4000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 00/0 : 43[63000] -> 44[4000] [receive] via NET/IB/0 comm 0x2b01e0000ab0 nRanks 96
e08r2n16:15465:18186 [1] NCCL INFO Channel 00 : 33[26000] -> 34[43000] via P2P/IPC comm 0x2ad418000ab0 nRanks 96
e08r3n02:14037:16485 [2] NCCL INFO Channel 00 : 58[43000] -> 59[63000] via P2P/IPC comm 0x2aef3c000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 00/0 : 67[63000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7678000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 00/0 : 39[63000] -> 40[4000] [receive] via NET/IB/0 comm 0x2acbdc000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 00/0 : 71[63000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b2a74000ab0 nRanks 96
e08r2n19:10428:13638 [1] NCCL INFO Channel 00 : 45[26000] -> 46[43000] via P2P/IPC comm 0x2b35b4000ab0 nRanks 96
e08r2n16:15466:18183 [2] NCCL INFO Channel 00 : 34[43000] -> 35[63000] via P2P/IPC comm 0x2bb254000ab0 nRanks 96
e08r3n02:14036:16488 [1] NCCL INFO Channel 00 : 57[26000] -> 58[43000] via P2P/IPC comm 0x2b0654000ab0 nRanks 96
e08r3n06:24083:27040 [1] NCCL INFO Channel 00 : 73[26000] -> 74[43000] via P2P/IPC comm 0x2ad250000ab0 nRanks 96
e08r3n01:21695:24041 [2] NCCL INFO Channel 01 : 54[43000] -> 55[63000] via P2P/IPC comm 0x2ad7ac000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 00/0 : 75[63000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab4d0000ab0 nRanks 96
e08r3n11:31583:2060 [0] NCCL INFO Channel 00/0 : 91[63000] -> 92[4000] [receive] via NET/IB/0 comm 0x2af4fc000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 00/0 : 83[63000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae4c4000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 00/0 : 79[63000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac910000ab0 nRanks 96
e08r2n15:23879:26734 [1] NCCL INFO Channel 01 : 29[26000] -> 30[43000] via P2P/IPC comm 0x2b0b84000ab0 nRanks 96
e08r2n17:22822:25590 [1] NCCL INFO Channel 00 : 37[26000] -> 38[43000] via P2P/IPC comm 0x2b7e78000ab0 nRanks 96
e08r2n19:10428:13638 [1] NCCL INFO Channel 01 : 45[26000] -> 46[43000] via P2P/IPC comm 0x2b35b4000ab0 nRanks 96
e08r2n16:15466:18183 [2] NCCL INFO Channel 01 : 34[43000] -> 35[63000] via P2P/IPC comm 0x2bb254000ab0 nRanks 96
e08r3n02:14037:16485 [2] NCCL INFO Channel 01 : 58[43000] -> 59[63000] via P2P/IPC comm 0x2aef3c000ab0 nRanks 96
e08r3n06:24084:27037 [2] NCCL INFO Channel 00 : 74[43000] -> 75[63000] via P2P/IPC comm 0x2b7b74000ab0 nRanks 96
e08r3n07:20447:22839 [1] NCCL INFO Channel 00 : 77[26000] -> 78[43000] via P2P/IPC comm 0x2b8f74000ab0 nRanks 96
e08r3n09:10074:13077 [1] NCCL INFO Channel 00 : 85[26000] -> 86[43000] via P2P/IPC comm 0x2aef78000ab0 nRanks 96
e08r2n15:23880:26728 [2] NCCL INFO Channel 00 : 30[43000] -> 31[63000] via P2P/IPC comm 0x2b2058000ab0 nRanks 96
e08r2n17:22823:25587 [2] NCCL INFO Channel 00 : 38[43000] -> 39[63000] via P2P/IPC comm 0x2afbe4000ab0 nRanks 96
e08r3n04:31688:1950 [1] NCCL INFO Channel 01 : 65[26000] -> 66[43000] via P2P/IPC comm 0x2af494000ab0 nRanks 96
e08r3n03:24232:27074 [2] NCCL INFO Channel 01 : 62[43000] -> 63[63000] via P2P/IPC comm 0x2acfe4000ab0 nRanks 96
e08r3n05:26317:29374 [1] NCCL INFO Channel 00 : 69[26000] -> 70[43000] via P2P/IPC comm 0x2b7ef4000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 00/0 : 87[63000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b66fc000ab0 nRanks 96
e08r2n18:2636:5675 [1] NCCL INFO Channel 00 : 41[26000] -> 42[43000] via P2P/IPC comm 0x2ac7e4000ab0 nRanks 96
e08r2n08:11393:19499 [0] NCCL INFO Channel 00/0 : 95[63000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b0148000ab0 nRanks 96
e08r2n08:11395:19506 [1] NCCL INFO Channel 00 : 1[26000] -> 2[43000] via P2P/IPC comm 0x2b23f8000ab0 nRanks 96
e08r2n16:15465:18186 [1] NCCL INFO Channel 01 : 33[26000] -> 34[43000] via P2P/IPC comm 0x2ad418000ab0 nRanks 96
e08r3n02:14036:16488 [1] NCCL INFO Channel 01 : 57[26000] -> 58[43000] via P2P/IPC comm 0x2b0654000ab0 nRanks 96
e08r3n06:24083:27040 [1] NCCL INFO Channel 01 : 73[26000] -> 74[43000] via P2P/IPC comm 0x2ad250000ab0 nRanks 96
e08r3n07:20448:22834 [2] NCCL INFO Channel 00 : 78[43000] -> 79[63000] via P2P/IPC comm 0x2b46e4000ab0 nRanks 96
e08r2n15:23880:26728 [2] NCCL INFO Channel 01 : 30[43000] -> 31[63000] via P2P/IPC comm 0x2b2058000ab0 nRanks 96
e08r2n17:22823:25587 [2] NCCL INFO Channel 01 : 38[43000] -> 39[63000] via P2P/IPC comm 0x2afbe4000ab0 nRanks 96
e08r3n04:31689:1947 [2] NCCL INFO Channel 01 : 66[43000] -> 67[63000] via P2P/IPC comm 0x2ad6c0000ab0 nRanks 96
e08r3n03:24231:27079 [1] NCCL INFO Channel 01 : 61[26000] -> 62[43000] via P2P/IPC comm 0x2b32ac000ab0 nRanks 96
e08r3n05:26318:29369 [2] NCCL INFO Channel 00 : 70[43000] -> 71[63000] via P2P/IPC comm 0x2b6538000ab0 nRanks 96
e08r3n10:20258:23243 [1] NCCL INFO Channel 00 : 89[26000] -> 90[43000] via P2P/IPC comm 0x2b150c000ab0 nRanks 96
e08r2n18:2637:5674 [2] NCCL INFO Channel 00 : 42[43000] -> 43[63000] via P2P/IPC comm 0x2b3950000ab0 nRanks 96
e08r3n11:31585:2059 [1] NCCL INFO Channel 00 : 93[26000] -> 94[43000] via P2P/IPC comm 0x2aed10000ab0 nRanks 96
e08r3n08:14729:17767 [1] NCCL INFO Channel 00 : 81[26000] -> 82[43000] via P2P/IPC comm 0x2b1500000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 00/0 : 11[63000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6ff4000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 00/0 : 7[63000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b795c000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 00/0 : 3[63000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af480000ab0 nRanks 96
e08r2n08:11396:19502 [2] NCCL INFO Channel 00 : 2[43000] -> 3[63000] via P2P/IPC comm 0x2b5920000ab0 nRanks 96
e08r2n08:11396:19502 [2] NCCL INFO Channel 01 : 2[43000] -> 3[63000] via P2P/IPC comm 0x2b5920000ab0 nRanks 96
e08r2n08:11395:19506 [1] NCCL INFO Channel 01 : 1[26000] -> 2[43000] via P2P/IPC comm 0x2b23f8000ab0 nRanks 96
e08r3n06:24084:27037 [2] NCCL INFO Channel 01 : 74[43000] -> 75[63000] via P2P/IPC comm 0x2b7b74000ab0 nRanks 96
e08r3n07:20447:22839 [1] NCCL INFO Channel 01 : 77[26000] -> 78[43000] via P2P/IPC comm 0x2b8f74000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 00/0 : 27[63000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b24e8000ab0 nRanks 96
e08r2n17:22822:25590 [1] NCCL INFO Channel 01 : 37[26000] -> 38[43000] via P2P/IPC comm 0x2b7e78000ab0 nRanks 96
e08r3n05:26318:29369 [2] NCCL INFO Channel 01 : 70[43000] -> 71[63000] via P2P/IPC comm 0x2b6538000ab0 nRanks 96
e08r3n10:20258:23243 [1] NCCL INFO Channel 01 : 89[26000] -> 90[43000] via P2P/IPC comm 0x2b150c000ab0 nRanks 96
e08r2n18:2636:5675 [1] NCCL INFO Channel 01 : 41[26000] -> 42[43000] via P2P/IPC comm 0x2ac7e4000ab0 nRanks 96
e08r3n11:31586:2056 [2] NCCL INFO Channel 00 : 94[43000] -> 95[63000] via P2P/IPC comm 0x2b32c4000ab0 nRanks 96
e08r3n08:14729:17767 [1] NCCL INFO Channel 01 : 81[26000] -> 82[43000] via P2P/IPC comm 0x2b1500000ab0 nRanks 96
e08r2n11:7502:10718 [1] NCCL INFO Channel 00 : 13[26000] -> 14[43000] via P2P/IPC comm 0x2b7f44000ab0 nRanks 96
e08r2n10:2342:5288 [1] NCCL INFO Channel 00 : 9[26000] -> 10[43000] via P2P/IPC comm 0x2ac2d8000ab0 nRanks 96
e08r2n09:28729:31686 [1] NCCL INFO Channel 00 : 5[26000] -> 6[43000] via P2P/IPC comm 0x2b70dc000ab0 nRanks 96
e08r3n09:10075:13072 [2] NCCL INFO Channel 00 : 86[43000] -> 87[63000] via P2P/IPC comm 0x2ba030000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 00/0 : 15[63000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b64c4000ab0 nRanks 96
e08r3n07:20448:22834 [2] NCCL INFO Channel 01 : 78[43000] -> 79[63000] via P2P/IPC comm 0x2b46e4000ab0 nRanks 96
e08r3n05:26317:29374 [1] NCCL INFO Channel 01 : 69[26000] -> 70[43000] via P2P/IPC comm 0x2b7ef4000ab0 nRanks 96
e08r3n10:20259:23240 [2] NCCL INFO Channel 00 : 90[43000] -> 91[63000] via P2P/IPC comm 0x2b85ec000ab0 nRanks 96
e08r3n11:31585:2059 [1] NCCL INFO Channel 01 : 93[26000] -> 94[43000] via P2P/IPC comm 0x2aed10000ab0 nRanks 96
e08r2n11:7503:10713 [2] NCCL INFO Channel 00 : 14[43000] -> 15[63000] via P2P/IPC comm 0x2b520c000ab0 nRanks 96
e08r2n10:2343:5283 [2] NCCL INFO Channel 00 : 10[43000] -> 11[63000] via P2P/IPC comm 0x2b1738000ab0 nRanks 96
e08r2n09:28730:31687 [2] NCCL INFO Channel 00 : 6[43000] -> 7[63000] via P2P/IPC comm 0x2b2864000ab0 nRanks 96
e08r3n09:10074:13077 [1] NCCL INFO Channel 01 : 85[26000] -> 86[43000] via P2P/IPC comm 0x2aef78000ab0 nRanks 96
e08r2n12:10510:13650 [2] NCCL INFO Channel 00 : 18[43000] -> 19[63000] via P2P/IPC comm 0x2b61d0000ab0 nRanks 96
e08r2n14:20094:23169 [1] NCCL INFO Channel 00 : 25[26000] -> 26[43000] via P2P/IPC comm 0x2b5774000ab0 nRanks 96
e08r2n13:26094:29027 [1] NCCL INFO Channel 00 : 21[26000] -> 22[43000] via P2P/IPC comm 0x2b2f78000ab0 nRanks 96
e08r3n10:20259:23240 [2] NCCL INFO Channel 01 : 90[43000] -> 91[63000] via P2P/IPC comm 0x2b85ec000ab0 nRanks 96
e08r3n11:31586:2056 [2] NCCL INFO Channel 01 : 94[43000] -> 95[63000] via P2P/IPC comm 0x2b32c4000ab0 nRanks 96
e08r2n11:7503:10713 [2] NCCL INFO Channel 01 : 14[43000] -> 15[63000] via P2P/IPC comm 0x2b520c000ab0 nRanks 96
e08r2n10:2342:5288 [1] NCCL INFO Channel 01 : 9[26000] -> 10[43000] via P2P/IPC comm 0x2ac2d8000ab0 nRanks 96
e08r2n09:28729:31686 [1] NCCL INFO Channel 01 : 5[26000] -> 6[43000] via P2P/IPC comm 0x2b70dc000ab0 nRanks 96
e08r3n09:10075:13072 [2] NCCL INFO Channel 01 : 86[43000] -> 87[63000] via P2P/IPC comm 0x2ba030000ab0 nRanks 96
e08r2n12:10509:13654 [1] NCCL INFO Channel 00 : 17[26000] -> 18[43000] via P2P/IPC comm 0x2baf28000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 00/0 : 23[63000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aeefc000ab0 nRanks 96
e08r2n13:26095:29022 [2] NCCL INFO Channel 00 : 22[43000] -> 23[63000] via P2P/IPC comm 0x2b1ff8000ab0 nRanks 96
e08r2n11:7502:10718 [1] NCCL INFO Channel 01 : 13[26000] -> 14[43000] via P2P/IPC comm 0x2b7f44000ab0 nRanks 96
e08r2n10:2343:5283 [2] NCCL INFO Channel 01 : 10[43000] -> 11[63000] via P2P/IPC comm 0x2b1738000ab0 nRanks 96
e08r2n09:28730:31687 [2] NCCL INFO Channel 01 : 6[43000] -> 7[63000] via P2P/IPC comm 0x2b2864000ab0 nRanks 96
e08r2n12:10510:13650 [2] NCCL INFO Channel 01 : 18[43000] -> 19[63000] via P2P/IPC comm 0x2b61d0000ab0 nRanks 96
e08r2n14:20095:23164 [2] NCCL INFO Channel 00 : 26[43000] -> 27[63000] via P2P/IPC comm 0x2baeac000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 00/0 : 19[63000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b1c18000ab0 nRanks 96
e08r2n12:10509:13654 [1] NCCL INFO Channel 01 : 17[26000] -> 18[43000] via P2P/IPC comm 0x2baf28000ab0 nRanks 96
e08r2n14:20094:23169 [1] NCCL INFO Channel 01 : 25[26000] -> 26[43000] via P2P/IPC comm 0x2b5774000ab0 nRanks 96
e08r2n13:26094:29027 [1] NCCL INFO Channel 01 : 21[26000] -> 22[43000] via P2P/IPC comm 0x2b2f78000ab0 nRanks 96
e08r2n14:20095:23164 [2] NCCL INFO Channel 01 : 26[43000] -> 27[63000] via P2P/IPC comm 0x2baeac000ab0 nRanks 96
e08r2n13:26095:29022 [2] NCCL INFO Channel 01 : 22[43000] -> 23[63000] via P2P/IPC comm 0x2b1ff8000ab0 nRanks 96
e08r2n18:2637:5674 [2] NCCL INFO Channel 01 : 42[43000] -> 43[63000] via P2P/IPC comm 0x2b3950000ab0 nRanks 96
e08r3n08:14730:17765 [2] NCCL INFO Channel 00 : 82[43000] -> 83[63000] via P2P/IPC comm 0x2af108000ab0 nRanks 96
e08r3n08:14730:17765 [2] NCCL INFO Channel 01 : 82[43000] -> 83[63000] via P2P/IPC comm 0x2af108000ab0 nRanks 96
e08r3n00:29330:31833 [3] NCCL INFO Channel 00/0 : 51[63000] -> 52[4000] [send] via NET/IB/0 comm 0x2b0fd8000ab0 nRanks 96
e08r3n04:31690:1945 [3] NCCL INFO Channel 00/0 : 67[63000] -> 68[4000] [send] via NET/IB/0 comm 0x2ac844000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 01/0 : 47[63000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af400000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 00 : 48[4000] -> 49[26000] via P2P/IPC comm 0x2af400000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Channel 01/0 : 63[63000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b0b28000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Channel 00 : 64[4000] -> 65[26000] via P2P/IPC comm 0x2b0b28000ab0 nRanks 96
e08r3n01:21696:24040 [3] NCCL INFO Channel 00/0 : 55[63000] -> 56[4000] [send] via NET/IB/0 comm 0x2b9e90000ab0 nRanks 96
e08r2n17:22824:25586 [3] NCCL INFO Channel 00/0 : 39[63000] -> 40[4000] [send] via NET/IB/0 comm 0x2ba5b0000ab0 nRanks 96
e08r3n03:24233:27076 [3] NCCL INFO Channel 00/0 : 63[63000] -> 64[4000] [send] via NET/IB/0 comm 0x2b694c000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 01/0 : 83[63000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae4c4000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 00 : 84[4000] -> 85[26000] via P2P/IPC comm 0x2ae4c4000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 01/0 : 51[63000] -> 52[4000] [receive] via NET/IB/0 comm 0x2acf70000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 01/0 : 35[63000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b0528000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 00 : 52[4000] -> 53[26000] via P2P/IPC comm 0x2acf70000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 00 : 36[4000] -> 37[26000] via P2P/IPC comm 0x2b0528000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 01/0 : 59[63000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba6a8000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 00 : 60[4000] -> 61[26000] via P2P/IPC comm 0x2ba6a8000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 01/0 : 71[63000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b2a74000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 00 : 72[4000] -> 73[26000] via P2P/IPC comm 0x2b2a74000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 01 : 48[4000] -> 49[26000] via P2P/IPC comm 0x2af400000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Channel 01 : 64[4000] -> 65[26000] via P2P/IPC comm 0x2b0b28000ab0 nRanks 96
e08r3n02:14038:16484 [3] NCCL INFO Channel 00/0 : 59[63000] -> 60[4000] [send] via NET/IB/0 comm 0x2b2fa8000ab0 nRanks 96
e08r3n02:14034:16489 [0] NCCL INFO Channel 01/0 : 55[63000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b5a20000ab0 nRanks 96
e08r3n02:14034:16489 [0] NCCL INFO Channel 00 : 56[4000] -> 57[26000] via P2P/IPC comm 0x2b5a20000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 01/0 : 67[63000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7678000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 00 : 68[4000] -> 69[26000] via P2P/IPC comm 0x2b7678000ab0 nRanks 96
e08r2n15:23881:26730 [3] NCCL INFO Channel 00/0 : 31[63000] -> 32[4000] [send] via NET/IB/0 comm 0x2b8004000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 01 : 52[4000] -> 53[26000] via P2P/IPC comm 0x2acf70000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 01 : 84[4000] -> 85[26000] via P2P/IPC comm 0x2ae4c4000ab0 nRanks 96
e08r3n08:14731:17766 [3] NCCL INFO Channel 00/0 : 83[63000] -> 84[4000] [send] via NET/IB/0 comm 0x2b9c40000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 01 : 36[4000] -> 37[26000] via P2P/IPC comm 0x2b0528000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 01 : 72[4000] -> 73[26000] via P2P/IPC comm 0x2b2a74000ab0 nRanks 96
e08r2n19:10430:13635 [3] NCCL INFO Channel 00/0 : 47[63000] -> 48[4000] [send] via NET/IB/0 comm 0x2b6b98000ab0 nRanks 96
e08r3n10:20260:23238 [3] NCCL INFO Channel 00/0 : 91[63000] -> 92[4000] [send] via NET/IB/0 comm 0x2ad094000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 01/0 : 87[63000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b66fc000ab0 nRanks 96
e08r2n18:2638:5671 [3] NCCL INFO Channel 00/0 : 43[63000] -> 44[4000] [send] via NET/IB/0 comm 0x2acc04000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 00 : 88[4000] -> 89[26000] via P2P/IPC comm 0x2b66fc000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 01/0 : 39[63000] -> 40[4000] [receive] via NET/IB/0 comm 0x2acbdc000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 01/0 : 43[63000] -> 44[4000] [receive] via NET/IB/0 comm 0x2b01e0000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 01/0 : 79[63000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac910000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 01/0 : 3[63000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af480000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 00 : 4[4000] -> 5[26000] via P2P/IPC comm 0x2af480000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 01 : 68[4000] -> 69[26000] via P2P/IPC comm 0x2b7678000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 00 : 40[4000] -> 41[26000] via P2P/IPC comm 0x2acbdc000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 00 : 44[4000] -> 45[26000] via P2P/IPC comm 0x2b01e0000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 00 : 80[4000] -> 81[26000] via P2P/IPC comm 0x2ac910000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 01 : 60[4000] -> 61[26000] via P2P/IPC comm 0x2ba6a8000ab0 nRanks 96
e08r3n02:14034:16489 [0] NCCL INFO Channel 01 : 56[4000] -> 57[26000] via P2P/IPC comm 0x2b5a20000ab0 nRanks 96
e08r2n16:15467:18182 [3] NCCL INFO Channel 00/0 : 35[63000] -> 36[4000] [send] via NET/IB/0 comm 0x2b2028000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 01/0 : 31[63000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af288000ab0 nRanks 96
e08r3n11:31587:2055 [3] NCCL INFO Channel 00/0 : 95[63000] -> 0[4000] [send] via NET/IB/0 comm 0x2b9048000ab0 nRanks 96
e08r3n07:20449:22835 [3] NCCL INFO Channel 00/0 : 79[63000] -> 80[4000] [send] via NET/IB/0 comm 0x2b27d8000ab0 nRanks 96
e08r2n12:10511:13649 [3] NCCL INFO Channel 00/0 : 19[63000] -> 20[4000] [send] via NET/IB/0 comm 0x2b408c000ab0 nRanks 96
e08r2n10:2344:5284 [3] NCCL INFO Channel 00/0 : 11[63000] -> 12[4000] [send] via NET/IB/0 comm 0x2b4c80000ab0 nRanks 96
e08r2n08:11397:19501 [3] NCCL INFO Channel 00/0 : 3[63000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6c88000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 00 : 32[4000] -> 33[26000] via P2P/IPC comm 0x2af288000ab0 nRanks 96
e08r3n11:31583:2060 [0] NCCL INFO Channel 01/0 : 91[63000] -> 92[4000] [receive] via NET/IB/0 comm 0x2af4fc000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 01/0 : 75[63000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab4d0000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 01/0 : 15[63000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b64c4000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 01/0 : 7[63000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b795c000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 01 : 44[4000] -> 45[26000] via P2P/IPC comm 0x2b01e0000ab0 nRanks 96
e08r3n09:10076:13074 [3] NCCL INFO Channel 00/0 : 87[63000] -> 88[4000] [send] via NET/IB/0 comm 0x2ab9b8000ab0 nRanks 96
e08r2n08:11393:19499 [0] NCCL INFO Channel 01/0 : 95[63000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b0148000ab0 nRanks 96
e08r2n08:11393:19499 [0] NCCL INFO Channel 00 : 0[4000] -> 1[26000] via P2P/IPC comm 0x2b0148000ab0 nRanks 96
e08r3n11:31583:2060 [0] NCCL INFO Channel 00 : 92[4000] -> 93[26000] via P2P/IPC comm 0x2af4fc000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 00 : 76[4000] -> 77[26000] via P2P/IPC comm 0x2ab4d0000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 00 : 16[4000] -> 17[26000] via P2P/IPC comm 0x2b64c4000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 00 : 8[4000] -> 9[26000] via P2P/IPC comm 0x2b795c000ab0 nRanks 96
e08r3n06:24085:27036 [3] NCCL INFO Channel 00/0 : 75[63000] -> 76[4000] [send] via NET/IB/0 comm 0x2b716c000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 01 : 80[4000] -> 81[26000] via P2P/IPC comm 0x2ac910000ab0 nRanks 96
e08r2n11:7504:10715 [3] NCCL INFO Channel 00/0 : 15[63000] -> 16[4000] [send] via NET/IB/0 comm 0x2bac84000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 01 : 4[4000] -> 5[26000] via P2P/IPC comm 0x2af480000ab0 nRanks 96
e08r3n05:26319:29370 [3] NCCL INFO Channel 00/0 : 71[63000] -> 72[4000] [send] via NET/IB/0 comm 0x2ae434000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 01 : 88[4000] -> 89[26000] via P2P/IPC comm 0x2b66fc000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 01/0 : 19[63000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b1c18000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 00 : 20[4000] -> 21[26000] via P2P/IPC comm 0x2b1c18000ab0 nRanks 96
e08r3n11:31583:2060 [0] NCCL INFO Channel 01 : 92[4000] -> 93[26000] via P2P/IPC comm 0x2af4fc000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 01 : 76[4000] -> 77[26000] via P2P/IPC comm 0x2ab4d0000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 01 : 16[4000] -> 17[26000] via P2P/IPC comm 0x2b64c4000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 01/0 : 11[63000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6ff4000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 01 : 40[4000] -> 41[26000] via P2P/IPC comm 0x2acbdc000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 01 : 32[4000] -> 33[26000] via P2P/IPC comm 0x2af288000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 01/0 : 27[63000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b24e8000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 00 : 28[4000] -> 29[26000] via P2P/IPC comm 0x2b24e8000ab0 nRanks 96
e08r2n08:11393:19499 [0] NCCL INFO Channel 01 : 0[4000] -> 1[26000] via P2P/IPC comm 0x2b0148000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 00 : 12[4000] -> 13[26000] via P2P/IPC comm 0x2b6ff4000ab0 nRanks 96
e08r2n09:28731:31688 [3] NCCL INFO Channel 00/0 : 7[63000] -> 8[4000] [send] via NET/IB/0 comm 0x2b6a54000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 01 : 8[4000] -> 9[26000] via P2P/IPC comm 0x2b795c000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 01 : 12[4000] -> 13[26000] via P2P/IPC comm 0x2b6ff4000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 01 : 20[4000] -> 21[26000] via P2P/IPC comm 0x2b1c18000ab0 nRanks 96
e08r2n14:20096:23166 [3] NCCL INFO Channel 00/0 : 27[63000] -> 28[4000] [send] via NET/IB/0 comm 0x2b60ec000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 01/0 : 23[63000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aeefc000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 01 : 28[4000] -> 29[26000] via P2P/IPC comm 0x2b24e8000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 00 : 24[4000] -> 25[26000] via P2P/IPC comm 0x2aeefc000ab0 nRanks 96
e08r2n13:26096:29023 [3] NCCL INFO Channel 00/0 : 23[63000] -> 24[4000] [send] via NET/IB/0 comm 0x2b8628000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 01 : 24[4000] -> 25[26000] via P2P/IPC comm 0x2aeefc000ab0 nRanks 96
e08r3n04:31688:1950 [1] NCCL INFO Connected all rings comm 0x2af494000ab0 nRanks 96 busId 26000
e08r3n00:29327:31836 [1] NCCL INFO Connected all rings comm 0x2b1704000ab0 nRanks 96 busId 26000
e08r2n17:22822:25590 [1] NCCL INFO Connected all rings comm 0x2b7e78000ab0 nRanks 96 busId 26000
e08r3n09:10074:13077 [1] NCCL INFO Connected all rings comm 0x2aef78000ab0 nRanks 96 busId 26000
e08r3n01:21694:24038 [1] NCCL INFO Connected all rings comm 0x2b9438000ab0 nRanks 96 busId 26000
e08r3n06:24083:27040 [1] NCCL INFO Connected all rings comm 0x2ad250000ab0 nRanks 96 busId 26000
e08r3n05:26317:29374 [1] NCCL INFO Connected all rings comm 0x2b7ef4000ab0 nRanks 96 busId 26000
e08r3n04:31688:1950 [1] NCCL INFO Channel 00/0 : 32[4000] -> 65[26000] [receive] via NET/IB/0 comm 0x2af494000ab0 nRanks 96
e08r3n03:24231:27079 [1] NCCL INFO Connected all rings comm 0x2b32ac000ab0 nRanks 96 busId 26000
e08r3n02:14036:16488 [1] NCCL INFO Connected all rings comm 0x2b0654000ab0 nRanks 96 busId 26000
e08r3n00:29327:31836 [1] NCCL INFO Channel 00/0 : 40[4000] -> 49[26000] [receive] via NET/IB/0 comm 0x2b1704000ab0 nRanks 96
e08r2n17:22822:25590 [1] NCCL INFO Channel 01/0 : 37[26000] -> 40[4000] [send] via NET/IB/0 comm 0x2b7e78000ab0 nRanks 96
e08r3n09:10074:13077 [1] NCCL INFO Channel 01/0 : 85[26000] -> 88[4000] [send] via NET/IB/0 comm 0x2aef78000ab0 nRanks 96
e08r3n01:21694:24038 [1] NCCL INFO Channel 01/0 : 53[26000] -> 56[4000] [send] via NET/IB/0 comm 0x2b9438000ab0 nRanks 96
e08r3n08:14729:17767 [1] NCCL INFO Connected all rings comm 0x2b1500000ab0 nRanks 96 busId 26000
e08r2n19:10428:13638 [1] NCCL INFO Connected all rings comm 0x2b35b4000ab0 nRanks 96 busId 26000
e08r3n06:24083:27040 [1] NCCL INFO Channel 00/0 : 68[4000] -> 73[26000] [receive] via NET/IB/0 comm 0x2ad250000ab0 nRanks 96
e08r3n05:26317:29374 [1] NCCL INFO Channel 01/0 : 69[26000] -> 72[4000] [send] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e08r3n03:24231:27079 [1] NCCL INFO Channel 01/0 : 61[26000] -> 76[4000] [send] via NET/IB/0 comm 0x2b32ac000ab0 nRanks 96
e08r3n10:20258:23243 [1] NCCL INFO Connected all rings comm 0x2b150c000ab0 nRanks 96 busId 26000
e08r2n16:15465:18186 [1] NCCL INFO Connected all rings comm 0x2ad418000ab0 nRanks 96 busId 26000
e08r3n11:31585:2059 [1] NCCL INFO Connected all rings comm 0x2aed10000ab0 nRanks 96 busId 26000
e08r3n02:14036:16488 [1] NCCL INFO Channel 00/0 : 52[4000] -> 57[26000] [receive] via NET/IB/0 comm 0x2b0654000ab0 nRanks 96
e08r2n18:2636:5675 [1] NCCL INFO Connected all rings comm 0x2ac7e4000ab0 nRanks 96 busId 26000
e08r2n08:11395:19506 [1] NCCL INFO Connected all rings comm 0x2b23f8000ab0 nRanks 96 busId 26000
e08r2n12:10509:13654 [1] NCCL INFO Connected all rings comm 0x2baf28000ab0 nRanks 96 busId 26000
e08r2n19:10428:13638 [1] NCCL INFO Channel 01/0 : 45[26000] -> 52[4000] [send] via NET/IB/0 comm 0x2b35b4000ab0 nRanks 96
e08r2n11:7502:10718 [1] NCCL INFO Connected all rings comm 0x2b7f44000ab0 nRanks 96 busId 26000
e08r3n08:14729:17767 [1] NCCL INFO Channel 00/0 : 72[4000] -> 81[26000] [receive] via NET/IB/0 comm 0x2b1500000ab0 nRanks 96
e08r2n10:2342:5288 [1] NCCL INFO Connected all rings comm 0x2ac2d8000ab0 nRanks 96 busId 26000
e08r2n16:15465:18186 [1] NCCL INFO Channel 00/0 : 16[4000] -> 33[26000] [receive] via NET/IB/0 comm 0x2ad418000ab0 nRanks 96
e08r2n13:26094:29027 [1] NCCL INFO Connected all rings comm 0x2b2f78000ab0 nRanks 96 busId 26000
e08r3n10:20258:23243 [1] NCCL INFO Channel 00/0 : 84[4000] -> 89[26000] [receive] via NET/IB/0 comm 0x2b150c000ab0 nRanks 96
e08r3n11:31585:2059 [1] NCCL INFO Channel 00 : 93[26000] -> 92[4000] via P2P/IPC comm 0x2aed10000ab0 nRanks 96
e08r2n18:2636:5675 [1] NCCL INFO Channel 00/0 : 36[4000] -> 41[26000] [receive] via NET/IB/0 comm 0x2ac7e4000ab0 nRanks 96
e08r2n11:7502:10718 [1] NCCL INFO Channel 01/0 : 13[26000] -> 20[4000] [send] via NET/IB/0 comm 0x2b7f44000ab0 nRanks 96
e08r2n12:10509:13654 [1] NCCL INFO Channel 00/0 : 8[4000] -> 17[26000] [receive] via NET/IB/0 comm 0x2baf28000ab0 nRanks 96
e08r2n15:23879:26734 [1] NCCL INFO Connected all rings comm 0x2b0b84000ab0 nRanks 96 busId 26000
e08r2n08:11395:19506 [1] NCCL INFO Channel 00 : 1[26000] -> 0[4000] via P2P/IPC comm 0x2b23f8000ab0 nRanks 96
e08r2n10:2342:5288 [1] NCCL INFO Channel 00/0 : 4[4000] -> 9[26000] [receive] via NET/IB/0 comm 0x2ac2d8000ab0 nRanks 96
e08r2n13:26094:29027 [1] NCCL INFO Channel 01/0 : 21[26000] -> 24[4000] [send] via NET/IB/0 comm 0x2b2f78000ab0 nRanks 96
e08r3n11:31585:2059 [1] NCCL INFO Channel 01 : 93[26000] -> 92[4000] via P2P/IPC comm 0x2aed10000ab0 nRanks 96
e08r2n08:11395:19506 [1] NCCL INFO Channel 01 : 1[26000] -> 0[4000] via P2P/IPC comm 0x2b23f8000ab0 nRanks 96
e08r3n07:20447:22839 [1] NCCL INFO Connected all rings comm 0x2b8f74000ab0 nRanks 96 busId 26000
e08r2n14:20094:23169 [1] NCCL INFO Connected all rings comm 0x2b5774000ab0 nRanks 96 busId 26000
e08r2n15:23879:26734 [1] NCCL INFO Channel 01/0 : 29[26000] -> 60[4000] [send] via NET/IB/0 comm 0x2b0b84000ab0 nRanks 96
e08r2n09:28729:31686 [1] NCCL INFO Connected all rings comm 0x2b70dc000ab0 nRanks 96 busId 26000
e08r3n07:20447:22839 [1] NCCL INFO Channel 01/0 : 77[26000] -> 84[4000] [send] via NET/IB/0 comm 0x2b8f74000ab0 nRanks 96
e08r2n14:20094:23169 [1] NCCL INFO Channel 00/0 : 20[4000] -> 25[26000] [receive] via NET/IB/0 comm 0x2b5774000ab0 nRanks 96
e08r2n09:28729:31686 [1] NCCL INFO Channel 01/0 : 5[26000] -> 8[4000] [send] via NET/IB/0 comm 0x2b70dc000ab0 nRanks 96
e08r3n04:31690:1945 [3] NCCL INFO Channel 01/0 : 67[63000] -> 68[4000] [send] via NET/IB/0 comm 0x2ac844000ab0 nRanks 96
e08r3n00:29330:31833 [3] NCCL INFO Channel 01/0 : 51[63000] -> 52[4000] [send] via NET/IB/0 comm 0x2b0fd8000ab0 nRanks 96
e08r3n01:21696:24040 [3] NCCL INFO Channel 01/0 : 55[63000] -> 56[4000] [send] via NET/IB/0 comm 0x2b9e90000ab0 nRanks 96
e08r2n17:22824:25586 [3] NCCL INFO Channel 01/0 : 39[63000] -> 40[4000] [send] via NET/IB/0 comm 0x2ba5b0000ab0 nRanks 96
e08r3n03:24233:27076 [3] NCCL INFO Channel 01/0 : 63[63000] -> 64[4000] [send] via NET/IB/0 comm 0x2b694c000ab0 nRanks 96
e08r3n02:14038:16484 [3] NCCL INFO Channel 01/0 : 59[63000] -> 60[4000] [send] via NET/IB/0 comm 0x2b2fa8000ab0 nRanks 96
e08r2n15:23881:26730 [3] NCCL INFO Channel 01/0 : 31[63000] -> 32[4000] [send] via NET/IB/0 comm 0x2b8004000ab0 nRanks 96
e08r3n08:14731:17766 [3] NCCL INFO Channel 01/0 : 83[63000] -> 84[4000] [send] via NET/IB/0 comm 0x2b9c40000ab0 nRanks 96
e08r3n10:20260:23238 [3] NCCL INFO Channel 01/0 : 91[63000] -> 92[4000] [send] via NET/IB/0 comm 0x2ad094000ab0 nRanks 96
e08r2n19:10430:13635 [3] NCCL INFO Channel 01/0 : 47[63000] -> 48[4000] [send] via NET/IB/0 comm 0x2b6b98000ab0 nRanks 96
e08r2n18:2638:5671 [3] NCCL INFO Channel 01/0 : 43[63000] -> 44[4000] [send] via NET/IB/0 comm 0x2acc04000ab0 nRanks 96
e08r2n16:15467:18182 [3] NCCL INFO Channel 01/0 : 35[63000] -> 36[4000] [send] via NET/IB/0 comm 0x2b2028000ab0 nRanks 96
e08r3n11:31587:2055 [3] NCCL INFO Channel 01/0 : 95[63000] -> 0[4000] [send] via NET/IB/0 comm 0x2b9048000ab0 nRanks 96
e08r3n07:20449:22835 [3] NCCL INFO Channel 01/0 : 79[63000] -> 80[4000] [send] via NET/IB/0 comm 0x2b27d8000ab0 nRanks 96
e08r2n10:2344:5284 [3] NCCL INFO Channel 01/0 : 11[63000] -> 12[4000] [send] via NET/IB/0 comm 0x2b4c80000ab0 nRanks 96
e08r2n12:10511:13649 [3] NCCL INFO Channel 01/0 : 19[63000] -> 20[4000] [send] via NET/IB/0 comm 0x2b408c000ab0 nRanks 96
e08r2n08:11397:19501 [3] NCCL INFO Channel 01/0 : 3[63000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6c88000ab0 nRanks 96
e08r3n09:10076:13074 [3] NCCL INFO Channel 01/0 : 87[63000] -> 88[4000] [send] via NET/IB/0 comm 0x2ab9b8000ab0 nRanks 96
e08r3n06:24085:27036 [3] NCCL INFO Channel 01/0 : 75[63000] -> 76[4000] [send] via NET/IB/0 comm 0x2b716c000ab0 nRanks 96
e08r2n11:7504:10715 [3] NCCL INFO Channel 01/0 : 15[63000] -> 16[4000] [send] via NET/IB/0 comm 0x2bac84000ab0 nRanks 96
e08r3n05:26319:29370 [3] NCCL INFO Channel 01/0 : 71[63000] -> 72[4000] [send] via NET/IB/0 comm 0x2ae434000ab0 nRanks 96
e08r2n09:28731:31688 [3] NCCL INFO Channel 01/0 : 7[63000] -> 8[4000] [send] via NET/IB/0 comm 0x2b6a54000ab0 nRanks 96
e08r2n14:20096:23166 [3] NCCL INFO Channel 01/0 : 27[63000] -> 28[4000] [send] via NET/IB/0 comm 0x2b60ec000ab0 nRanks 96
e08r3n00:29328:31837 [2] NCCL INFO Connected all rings comm 0x2ad7f0000ab0 nRanks 96 busId 43000
e08r3n04:31689:1947 [2] NCCL INFO Connected all rings comm 0x2ad6c0000ab0 nRanks 96 busId 43000
e08r3n01:21695:24041 [2] NCCL INFO Connected all rings comm 0x2ad7ac000ab0 nRanks 96 busId 43000
e08r2n13:26096:29023 [3] NCCL INFO Channel 01/0 : 23[63000] -> 24[4000] [send] via NET/IB/0 comm 0x2b8628000ab0 nRanks 96
e08r2n17:22823:25587 [2] NCCL INFO Connected all rings comm 0x2afbe4000ab0 nRanks 96 busId 43000
e08r3n02:14037:16485 [2] NCCL INFO Connected all rings comm 0x2aef3c000ab0 nRanks 96 busId 43000
e08r3n03:24232:27074 [2] NCCL INFO Connected all rings comm 0x2acfe4000ab0 nRanks 96 busId 43000
e08r3n00:29328:31837 [2] NCCL INFO Channel 00 : 50[43000] -> 49[26000] via P2P/IPC comm 0x2ad7f0000ab0 nRanks 96
e08r2n15:23880:26728 [2] NCCL INFO Connected all rings comm 0x2b2058000ab0 nRanks 96 busId 43000
e08r3n04:31689:1947 [2] NCCL INFO Channel 00 : 66[43000] -> 65[26000] via P2P/IPC comm 0x2ad6c0000ab0 nRanks 96
e08r2n18:2637:5674 [2] NCCL INFO Connected all rings comm 0x2b3950000ab0 nRanks 96 busId 43000
e08r3n08:14730:17765 [2] NCCL INFO Connected all rings comm 0x2af108000ab0 nRanks 96 busId 43000
e08r3n10:20259:23240 [2] NCCL INFO Connected all rings comm 0x2b85ec000ab0 nRanks 96 busId 43000
e08r2n19:10429:13634 [2] NCCL INFO Connected all rings comm 0x2b0404000ab0 nRanks 96 busId 43000
e08r3n00:29328:31837 [2] NCCL INFO Channel 01 : 50[43000] -> 49[26000] via P2P/IPC comm 0x2ad7f0000ab0 nRanks 96
e08r2n16:15466:18183 [2] NCCL INFO Connected all rings comm 0x2bb254000ab0 nRanks 96 busId 43000
e08r3n04:31689:1947 [2] NCCL INFO Channel 01 : 66[43000] -> 65[26000] via P2P/IPC comm 0x2ad6c0000ab0 nRanks 96
e08r3n01:21695:24041 [2] NCCL INFO Channel 00 : 54[43000] -> 53[26000] via P2P/IPC comm 0x2ad7ac000ab0 nRanks 96
e08r2n17:22823:25587 [2] NCCL INFO Channel 00 : 38[43000] -> 37[26000] via P2P/IPC comm 0x2afbe4000ab0 nRanks 96
e08r3n11:31586:2056 [2] NCCL INFO Connected all rings comm 0x2b32c4000ab0 nRanks 96 busId 43000
e08r3n03:24232:27074 [2] NCCL INFO Channel 00 : 62[43000] -> 61[26000] via P2P/IPC comm 0x2acfe4000ab0 nRanks 96
e08r3n02:14037:16485 [2] NCCL INFO Channel 00 : 58[43000] -> 57[26000] via P2P/IPC comm 0x2aef3c000ab0 nRanks 96
e08r3n09:10075:13072 [2] NCCL INFO Connected all rings comm 0x2ba030000ab0 nRanks 96 busId 43000
e08r3n07:20448:22834 [2] NCCL INFO Connected all rings comm 0x2b46e4000ab0 nRanks 96 busId 43000
e08r2n10:2343:5283 [2] NCCL INFO Connected all rings comm 0x2b1738000ab0 nRanks 96 busId 43000
e08r2n12:10510:13650 [2] NCCL INFO Connected all rings comm 0x2b61d0000ab0 nRanks 96 busId 43000
e08r3n01:21695:24041 [2] NCCL INFO Channel 01 : 54[43000] -> 53[26000] via P2P/IPC comm 0x2ad7ac000ab0 nRanks 96
e08r2n08:11396:19502 [2] NCCL INFO Connected all rings comm 0x2b5920000ab0 nRanks 96 busId 43000
e08r2n17:22823:25587 [2] NCCL INFO Channel 01 : 38[43000] -> 37[26000] via P2P/IPC comm 0x2afbe4000ab0 nRanks 96
e08r2n15:23880:26728 [2] NCCL INFO Channel 00 : 30[43000] -> 29[26000] via P2P/IPC comm 0x2b2058000ab0 nRanks 96
e08r3n05:26318:29369 [2] NCCL INFO Connected all rings comm 0x2b6538000ab0 nRanks 96 busId 43000
e08r2n18:2637:5674 [2] NCCL INFO Channel 00 : 42[43000] -> 41[26000] via P2P/IPC comm 0x2b3950000ab0 nRanks 96
e08r3n06:24084:27037 [2] NCCL INFO Connected all rings comm 0x2b7b74000ab0 nRanks 96 busId 43000
e08r3n03:24232:27074 [2] NCCL INFO Channel 01 : 62[43000] -> 61[26000] via P2P/IPC comm 0x2acfe4000ab0 nRanks 96
e08r2n11:7503:10713 [2] NCCL INFO Connected all rings comm 0x2b520c000ab0 nRanks 96 busId 43000
e08r3n10:20259:23240 [2] NCCL INFO Channel 00 : 90[43000] -> 89[26000] via P2P/IPC comm 0x2b85ec000ab0 nRanks 96
e08r3n02:14037:16485 [2] NCCL INFO Channel 01 : 58[43000] -> 57[26000] via P2P/IPC comm 0x2aef3c000ab0 nRanks 96
e08r2n19:10429:13634 [2] NCCL INFO Channel 00 : 46[43000] -> 45[26000] via P2P/IPC comm 0x2b0404000ab0 nRanks 96
e08r2n16:15466:18183 [2] NCCL INFO Channel 00 : 34[43000] -> 33[26000] via P2P/IPC comm 0x2bb254000ab0 nRanks 96
e08r3n08:14730:17765 [2] NCCL INFO Channel 00 : 82[43000] -> 81[26000] via P2P/IPC comm 0x2af108000ab0 nRanks 96
e08r2n15:23880:26728 [2] NCCL INFO Channel 01 : 30[43000] -> 29[26000] via P2P/IPC comm 0x2b2058000ab0 nRanks 96
e08r3n10:20259:23240 [2] NCCL INFO Channel 01 : 90[43000] -> 89[26000] via P2P/IPC comm 0x2b85ec000ab0 nRanks 96
e08r2n18:2637:5674 [2] NCCL INFO Channel 01 : 42[43000] -> 41[26000] via P2P/IPC comm 0x2b3950000ab0 nRanks 96
e08r2n19:10429:13634 [2] NCCL INFO Channel 01 : 46[43000] -> 45[26000] via P2P/IPC comm 0x2b0404000ab0 nRanks 96
e08r2n09:28730:31687 [2] NCCL INFO Connected all rings comm 0x2b2864000ab0 nRanks 96 busId 43000
e08r2n16:15466:18183 [2] NCCL INFO Channel 01 : 34[43000] -> 33[26000] via P2P/IPC comm 0x2bb254000ab0 nRanks 96
e08r3n09:10075:13072 [2] NCCL INFO Channel 00 : 86[43000] -> 85[26000] via P2P/IPC comm 0x2ba030000ab0 nRanks 96
e08r2n10:2343:5283 [2] NCCL INFO Channel 00 : 10[43000] -> 9[26000] via P2P/IPC comm 0x2b1738000ab0 nRanks 96
e08r3n11:31586:2056 [2] NCCL INFO Channel 00 : 94[43000] -> 93[26000] via P2P/IPC comm 0x2b32c4000ab0 nRanks 96
e08r3n08:14730:17765 [2] NCCL INFO Channel 01 : 82[43000] -> 81[26000] via P2P/IPC comm 0x2af108000ab0 nRanks 96
e08r3n07:20448:22834 [2] NCCL INFO Channel 00 : 78[43000] -> 77[26000] via P2P/IPC comm 0x2b46e4000ab0 nRanks 96
e08r2n12:10510:13650 [2] NCCL INFO Channel 00 : 18[43000] -> 17[26000] via P2P/IPC comm 0x2b61d0000ab0 nRanks 96
e08r3n06:24084:27037 [2] NCCL INFO Channel 00 : 74[43000] -> 73[26000] via P2P/IPC comm 0x2b7b74000ab0 nRanks 96
e08r2n08:11396:19502 [2] NCCL INFO Channel 00 : 2[43000] -> 1[26000] via P2P/IPC comm 0x2b5920000ab0 nRanks 96
e08r2n11:7503:10713 [2] NCCL INFO Channel 00 : 14[43000] -> 13[26000] via P2P/IPC comm 0x2b520c000ab0 nRanks 96
e08r2n10:2343:5283 [2] NCCL INFO Channel 01 : 10[43000] -> 9[26000] via P2P/IPC comm 0x2b1738000ab0 nRanks 96
e08r3n09:10075:13072 [2] NCCL INFO Channel 01 : 86[43000] -> 85[26000] via P2P/IPC comm 0x2ba030000ab0 nRanks 96
e08r3n05:26318:29369 [2] NCCL INFO Channel 00 : 70[43000] -> 69[26000] via P2P/IPC comm 0x2b6538000ab0 nRanks 96
e08r3n11:31586:2056 [2] NCCL INFO Channel 01 : 94[43000] -> 93[26000] via P2P/IPC comm 0x2b32c4000ab0 nRanks 96
e08r3n07:20448:22834 [2] NCCL INFO Channel 01 : 78[43000] -> 77[26000] via P2P/IPC comm 0x2b46e4000ab0 nRanks 96
e08r2n14:20095:23164 [2] NCCL INFO Connected all rings comm 0x2baeac000ab0 nRanks 96 busId 43000
e08r3n06:24084:27037 [2] NCCL INFO Channel 01 : 74[43000] -> 73[26000] via P2P/IPC comm 0x2b7b74000ab0 nRanks 96
e08r2n08:11396:19502 [2] NCCL INFO Channel 01 : 2[43000] -> 1[26000] via P2P/IPC comm 0x2b5920000ab0 nRanks 96
e08r2n11:7503:10713 [2] NCCL INFO Channel 01 : 14[43000] -> 13[26000] via P2P/IPC comm 0x2b520c000ab0 nRanks 96
e08r2n12:10510:13650 [2] NCCL INFO Channel 01 : 18[43000] -> 17[26000] via P2P/IPC comm 0x2b61d0000ab0 nRanks 96
e08r2n13:26095:29022 [2] NCCL INFO Connected all rings comm 0x2b1ff8000ab0 nRanks 96 busId 43000
e08r3n05:26318:29369 [2] NCCL INFO Channel 01 : 70[43000] -> 69[26000] via P2P/IPC comm 0x2b6538000ab0 nRanks 96
e08r2n09:28730:31687 [2] NCCL INFO Channel 00 : 6[43000] -> 5[26000] via P2P/IPC comm 0x2b2864000ab0 nRanks 96
e08r2n09:28730:31687 [2] NCCL INFO Channel 01 : 6[43000] -> 5[26000] via P2P/IPC comm 0x2b2864000ab0 nRanks 96
e08r2n14:20095:23164 [2] NCCL INFO Channel 00 : 26[43000] -> 25[26000] via P2P/IPC comm 0x2baeac000ab0 nRanks 96
e08r2n13:26095:29022 [2] NCCL INFO Channel 00 : 22[43000] -> 21[26000] via P2P/IPC comm 0x2b1ff8000ab0 nRanks 96
e08r2n14:20095:23164 [2] NCCL INFO Channel 01 : 26[43000] -> 25[26000] via P2P/IPC comm 0x2baeac000ab0 nRanks 96
e08r2n13:26095:29022 [2] NCCL INFO Channel 01 : 22[43000] -> 21[26000] via P2P/IPC comm 0x2b1ff8000ab0 nRanks 96
e08r3n04:31690:1945 [3] NCCL INFO Connected all rings comm 0x2ac844000ab0 nRanks 96 busId 63000
e08r3n04:31690:1945 [3] NCCL INFO Channel 00 : 67[63000] -> 66[43000] via P2P/IPC comm 0x2ac844000ab0 nRanks 96
e08r3n00:29330:31833 [3] NCCL INFO Connected all rings comm 0x2b0fd8000ab0 nRanks 96 busId 63000
e08r3n00:29330:31833 [3] NCCL INFO Channel 00 : 51[63000] -> 50[43000] via P2P/IPC comm 0x2b0fd8000ab0 nRanks 96
e08r3n04:31690:1945 [3] NCCL INFO Channel 01 : 67[63000] -> 66[43000] via P2P/IPC comm 0x2ac844000ab0 nRanks 96
e08r3n00:29330:31833 [3] NCCL INFO Channel 01 : 51[63000] -> 50[43000] via P2P/IPC comm 0x2b0fd8000ab0 nRanks 96
e08r2n18:2638:5671 [3] NCCL INFO Connected all rings comm 0x2acc04000ab0 nRanks 96 busId 63000
e08r2n18:2638:5671 [3] NCCL INFO Channel 00 : 43[63000] -> 42[43000] via P2P/IPC comm 0x2acc04000ab0 nRanks 96
e08r3n01:21696:24040 [3] NCCL INFO Connected all rings comm 0x2b9e90000ab0 nRanks 96 busId 63000
e08r3n01:21696:24040 [3] NCCL INFO Channel 00 : 55[63000] -> 54[43000] via P2P/IPC comm 0x2b9e90000ab0 nRanks 96
e08r3n03:24233:27076 [3] NCCL INFO Connected all rings comm 0x2b694c000ab0 nRanks 96 busId 63000
e08r3n03:24233:27076 [3] NCCL INFO Channel 00 : 63[63000] -> 62[43000] via P2P/IPC comm 0x2b694c000ab0 nRanks 96
e08r2n18:2638:5671 [3] NCCL INFO Channel 01 : 43[63000] -> 42[43000] via P2P/IPC comm 0x2acc04000ab0 nRanks 96
e08r3n11:31587:2055 [3] NCCL INFO Connected all rings comm 0x2b9048000ab0 nRanks 96 busId 63000
e08r3n11:31587:2055 [3] NCCL INFO Channel 00 : 95[63000] -> 94[43000] via P2P/IPC comm 0x2b9048000ab0 nRanks 96
e08r3n03:24233:27076 [3] NCCL INFO Channel 01 : 63[63000] -> 62[43000] via P2P/IPC comm 0x2b694c000ab0 nRanks 96
e08r2n16:15467:18182 [3] NCCL INFO Connected all rings comm 0x2b2028000ab0 nRanks 96 busId 63000
e08r3n01:21696:24040 [3] NCCL INFO Channel 01 : 55[63000] -> 54[43000] via P2P/IPC comm 0x2b9e90000ab0 nRanks 96
e08r2n19:10430:13635 [3] NCCL INFO Connected all rings comm 0x2b6b98000ab0 nRanks 96 busId 63000
e08r2n16:15467:18182 [3] NCCL INFO Channel 00 : 35[63000] -> 34[43000] via P2P/IPC comm 0x2b2028000ab0 nRanks 96
e08r3n08:14731:17766 [3] NCCL INFO Connected all rings comm 0x2b9c40000ab0 nRanks 96 busId 63000
e08r2n19:10430:13635 [3] NCCL INFO Channel 00 : 47[63000] -> 46[43000] via P2P/IPC comm 0x2b6b98000ab0 nRanks 96
e08r3n08:14731:17766 [3] NCCL INFO Channel 00 : 83[63000] -> 82[43000] via P2P/IPC comm 0x2b9c40000ab0 nRanks 96
e08r3n06:24085:27036 [3] NCCL INFO Connected all rings comm 0x2b716c000ab0 nRanks 96 busId 63000
e08r3n06:24085:27036 [3] NCCL INFO Channel 00 : 75[63000] -> 74[43000] via P2P/IPC comm 0x2b716c000ab0 nRanks 96
e08r2n11:7504:10715 [3] NCCL INFO Connected all rings comm 0x2bac84000ab0 nRanks 96 busId 63000
e08r2n11:7504:10715 [3] NCCL INFO Channel 00 : 15[63000] -> 14[43000] via P2P/IPC comm 0x2bac84000ab0 nRanks 96
e08r3n11:31587:2055 [3] NCCL INFO Channel 01 : 95[63000] -> 94[43000] via P2P/IPC comm 0x2b9048000ab0 nRanks 96
e08r3n02:14038:16484 [3] NCCL INFO Connected all rings comm 0x2b2fa8000ab0 nRanks 96 busId 63000
e08r3n02:14038:16484 [3] NCCL INFO Channel 00 : 59[63000] -> 58[43000] via P2P/IPC comm 0x2b2fa8000ab0 nRanks 96
e08r2n15:23881:26730 [3] NCCL INFO Connected all rings comm 0x2b8004000ab0 nRanks 96 busId 63000
e08r2n12:10511:13649 [3] NCCL INFO Connected all rings comm 0x2b408c000ab0 nRanks 96 busId 63000
e08r2n15:23881:26730 [3] NCCL INFO Channel 00 : 31[63000] -> 30[43000] via P2P/IPC comm 0x2b8004000ab0 nRanks 96
e08r2n12:10511:13649 [3] NCCL INFO Channel 00 : 19[63000] -> 18[43000] via P2P/IPC comm 0x2b408c000ab0 nRanks 96
e08r3n08:14731:17766 [3] NCCL INFO Channel 01 : 83[63000] -> 82[43000] via P2P/IPC comm 0x2b9c40000ab0 nRanks 96
e08r2n19:10430:13635 [3] NCCL INFO Channel 01 : 47[63000] -> 46[43000] via P2P/IPC comm 0x2b6b98000ab0 nRanks 96
e08r3n06:24085:27036 [3] NCCL INFO Channel 01 : 75[63000] -> 74[43000] via P2P/IPC comm 0x2b716c000ab0 nRanks 96
e08r2n11:7504:10715 [3] NCCL INFO Channel 01 : 15[63000] -> 14[43000] via P2P/IPC comm 0x2bac84000ab0 nRanks 96
e08r2n16:15467:18182 [3] NCCL INFO Channel 01 : 35[63000] -> 34[43000] via P2P/IPC comm 0x2b2028000ab0 nRanks 96
e08r3n05:26319:29370 [3] NCCL INFO Connected all rings comm 0x2ae434000ab0 nRanks 96 busId 63000
e08r3n05:26319:29370 [3] NCCL INFO Channel 00 : 71[63000] -> 70[43000] via P2P/IPC comm 0x2ae434000ab0 nRanks 96
e08r2n12:10511:13649 [3] NCCL INFO Channel 01 : 19[63000] -> 18[43000] via P2P/IPC comm 0x2b408c000ab0 nRanks 96
e08r2n15:23881:26730 [3] NCCL INFO Channel 01 : 31[63000] -> 30[43000] via P2P/IPC comm 0x2b8004000ab0 nRanks 96
e08r3n02:14038:16484 [3] NCCL INFO Channel 01 : 59[63000] -> 58[43000] via P2P/IPC comm 0x2b2fa8000ab0 nRanks 96
e08r2n14:20096:23166 [3] NCCL INFO Connected all rings comm 0x2b60ec000ab0 nRanks 96 busId 63000
e08r3n04:31690:1945 [3] NCCL INFO Connected all trees comm 0x2ac844000ab0 nRanks 96 busId 63000
e08r2n14:20096:23166 [3] NCCL INFO Channel 00 : 27[63000] -> 26[43000] via P2P/IPC comm 0x2b60ec000ab0 nRanks 96
e08r3n04:31690:1945 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n07:20449:22835 [3] NCCL INFO Connected all rings comm 0x2b27d8000ab0 nRanks 96 busId 63000
e08r3n04:31690:1945 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n07:20449:22835 [3] NCCL INFO Channel 00 : 79[63000] -> 78[43000] via P2P/IPC comm 0x2b27d8000ab0 nRanks 96
e08r2n08:11397:19501 [3] NCCL INFO Connected all rings comm 0x2b6c88000ab0 nRanks 96 busId 63000
e08r2n10:2344:5284 [3] NCCL INFO Connected all rings comm 0x2b4c80000ab0 nRanks 96 busId 63000
e08r2n08:11397:19501 [3] NCCL INFO Channel 00 : 3[63000] -> 2[43000] via P2P/IPC comm 0x2b6c88000ab0 nRanks 96
e08r2n10:2344:5284 [3] NCCL INFO Channel 00 : 11[63000] -> 10[43000] via P2P/IPC comm 0x2b4c80000ab0 nRanks 96
e08r3n09:10076:13074 [3] NCCL INFO Connected all rings comm 0x2ab9b8000ab0 nRanks 96 busId 63000
e08r3n00:29330:31833 [3] NCCL INFO Connected all trees comm 0x2b0fd8000ab0 nRanks 96 busId 63000
e08r3n09:10076:13074 [3] NCCL INFO Channel 00 : 87[63000] -> 86[43000] via P2P/IPC comm 0x2ab9b8000ab0 nRanks 96
e08r3n00:29330:31833 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n00:29330:31833 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26319:29370 [3] NCCL INFO Channel 01 : 71[63000] -> 70[43000] via P2P/IPC comm 0x2ae434000ab0 nRanks 96
e08r2n14:20096:23166 [3] NCCL INFO Channel 01 : 27[63000] -> 26[43000] via P2P/IPC comm 0x2b60ec000ab0 nRanks 96
e08r3n07:20449:22835 [3] NCCL INFO Channel 01 : 79[63000] -> 78[43000] via P2P/IPC comm 0x2b27d8000ab0 nRanks 96
e08r2n10:2344:5284 [3] NCCL INFO Channel 01 : 11[63000] -> 10[43000] via P2P/IPC comm 0x2b4c80000ab0 nRanks 96
e08r2n08:11397:19501 [3] NCCL INFO Channel 01 : 3[63000] -> 2[43000] via P2P/IPC comm 0x2b6c88000ab0 nRanks 96
e08r3n10:20260:23238 [3] NCCL INFO Connected all rings comm 0x2ad094000ab0 nRanks 96 busId 63000
e08r3n10:20260:23238 [3] NCCL INFO Channel 00 : 91[63000] -> 90[43000] via P2P/IPC comm 0x2ad094000ab0 nRanks 96
e08r3n09:10076:13074 [3] NCCL INFO Channel 01 : 87[63000] -> 86[43000] via P2P/IPC comm 0x2ab9b8000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Connected all rings comm 0x2acf70000ab0 nRanks 96 busId 4000
e08r3n10:20260:23238 [3] NCCL INFO Channel 01 : 91[63000] -> 90[43000] via P2P/IPC comm 0x2ad094000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Connected all rings comm 0x2b0b28000ab0 nRanks 96 busId 4000
e08r2n18:2638:5671 [3] NCCL INFO Connected all trees comm 0x2acc04000ab0 nRanks 96 busId 63000
e08r2n18:2638:5671 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2638:5671 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n08:11393:19499 [0] NCCL INFO Connected all rings comm 0x2b0148000ab0 nRanks 96 busId 4000
e08r3n02:14034:16489 [0] NCCL INFO Connected all rings comm 0x2b5a20000ab0 nRanks 96 busId 4000
e08r2n09:28731:31688 [3] NCCL INFO Connected all rings comm 0x2b6a54000ab0 nRanks 96 busId 63000
e08r2n09:28731:31688 [3] NCCL INFO Channel 00 : 7[63000] -> 6[43000] via P2P/IPC comm 0x2b6a54000ab0 nRanks 96
e08r3n03:24233:27076 [3] NCCL INFO Connected all trees comm 0x2b694c000ab0 nRanks 96 busId 63000
e08r3n07:20445:22838 [0] NCCL INFO Connected all rings comm 0x2ab4d0000ab0 nRanks 96 busId 4000
e08r3n03:24233:27076 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n03:24233:27076 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26096:29023 [3] NCCL INFO Connected all rings comm 0x2b8628000ab0 nRanks 96 busId 63000
e08r3n01:21696:24040 [3] NCCL INFO Connected all trees comm 0x2b9e90000ab0 nRanks 96 busId 63000
e08r2n13:26096:29023 [3] NCCL INFO Channel 00 : 23[63000] -> 22[43000] via P2P/IPC comm 0x2b8628000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Connected all rings comm 0x2af400000ab0 nRanks 96 busId 4000
e08r3n01:21696:24040 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n01:21696:24040 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10427:13639 [0] NCCL INFO Connected all rings comm 0x2b01e0000ab0 nRanks 96 busId 4000
e08r3n11:31587:2055 [3] NCCL INFO Connected all trees comm 0x2b9048000ab0 nRanks 96 busId 63000
e08r3n11:31587:2055 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n11:31587:2055 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26096:29023 [3] NCCL INFO Channel 01 : 23[63000] -> 22[43000] via P2P/IPC comm 0x2b8628000ab0 nRanks 96
e08r2n09:28731:31688 [3] NCCL INFO Channel 01 : 7[63000] -> 6[43000] via P2P/IPC comm 0x2b6a54000ab0 nRanks 96
e08r3n08:14731:17766 [3] NCCL INFO Connected all trees comm 0x2b9c40000ab0 nRanks 96 busId 63000
e08r2n16:15467:18182 [3] NCCL INFO Connected all trees comm 0x2b2028000ab0 nRanks 96 busId 63000
e08r3n03:24229:27078 [0] NCCL INFO Connected all rings comm 0x2ba6a8000ab0 nRanks 96 busId 4000
e08r2n19:10430:13635 [3] NCCL INFO Connected all trees comm 0x2b6b98000ab0 nRanks 96 busId 63000
e08r2n16:15467:18182 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14731:17766 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14731:17766 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15467:18182 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10430:13635 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10508:13653 [0] NCCL INFO Connected all rings comm 0x2b64c4000ab0 nRanks 96 busId 4000
e08r3n11:31586:2056 [2] NCCL INFO Connected all trees comm 0x2b32c4000ab0 nRanks 96 busId 43000
e08r2n16:15463:18187 [0] NCCL INFO Connected all rings comm 0x2af288000ab0 nRanks 96 busId 4000
e08r2n19:10430:13635 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10511:13649 [3] NCCL INFO Connected all trees comm 0x2b408c000ab0 nRanks 96 busId 63000
e08r3n11:31586:2056 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10511:13649 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n11:31586:2056 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14038:16484 [3] NCCL INFO Connected all trees comm 0x2b2fa8000ab0 nRanks 96 busId 63000
e08r2n11:7504:10715 [3] NCCL INFO Connected all trees comm 0x2bac84000ab0 nRanks 96 busId 63000
e08r2n15:23881:26730 [3] NCCL INFO Connected all trees comm 0x2b8004000ab0 nRanks 96 busId 63000
e08r3n06:24085:27036 [3] NCCL INFO Connected all trees comm 0x2b716c000ab0 nRanks 96 busId 63000
e08r2n12:10511:13649 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14038:16484 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n02:14038:16484 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7504:10715 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n11:7504:10715 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n15:23881:26730 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n06:24085:27036 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14727:17764 [0] NCCL INFO Connected all rings comm 0x2ac910000ab0 nRanks 96 busId 4000
e08r2n15:23881:26730 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n06:24085:27036 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10072:13076 [0] NCCL INFO Connected all rings comm 0x2ae4c4000ab0 nRanks 96 busId 4000
e08r3n05:26319:29370 [3] NCCL INFO Connected all trees comm 0x2ae434000ab0 nRanks 96 busId 63000
e08r3n05:26319:29370 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n05:26319:29370 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n07:20449:22835 [3] NCCL INFO Connected all trees comm 0x2b27d8000ab0 nRanks 96 busId 63000
e08r2n14:20096:23166 [3] NCCL INFO Connected all trees comm 0x2b60ec000ab0 nRanks 96 busId 63000
e08r3n07:20449:22835 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n07:20449:22835 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20096:23166 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n14:20096:23166 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22820:25591 [0] NCCL INFO Connected all rings comm 0x2b0528000ab0 nRanks 96 busId 4000
e08r2n09:28728:31689 [0] NCCL INFO Connected all rings comm 0x2af480000ab0 nRanks 96 busId 4000
e08r2n15:23877:26732 [0] NCCL INFO Connected all rings comm 0x2b24e8000ab0 nRanks 96 busId 4000
e08r3n11:31583:2060 [0] NCCL INFO Connected all rings comm 0x2af4fc000ab0 nRanks 96 busId 4000
e08r2n08:11397:19501 [3] NCCL INFO Connected all trees comm 0x2b6c88000ab0 nRanks 96 busId 63000
e08r3n09:10076:13074 [3] NCCL INFO Connected all trees comm 0x2ab9b8000ab0 nRanks 96 busId 63000
e08r2n08:11397:19501 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11397:19501 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2344:5284 [3] NCCL INFO Connected all trees comm 0x2b4c80000ab0 nRanks 96 busId 63000
e08r3n09:10076:13074 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n09:10076:13074 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2344:5284 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2344:5284 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26093:29026 [0] NCCL INFO Connected all rings comm 0x2b1c18000ab0 nRanks 96 busId 4000
e08r3n10:20260:23238 [3] NCCL INFO Connected all trees comm 0x2ad094000ab0 nRanks 96 busId 63000
e08r3n10:20260:23238 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n10:20260:23238 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n08:11396:19502 [2] NCCL INFO Connected all trees comm 0x2b5920000ab0 nRanks 96 busId 43000
e08r2n08:11396:19502 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11396:19502 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26315:29373 [0] NCCL INFO Connected all rings comm 0x2b7678000ab0 nRanks 96 busId 4000
e08r3n06:24081:27041 [0] NCCL INFO Connected all rings comm 0x2b2a74000ab0 nRanks 96 busId 4000
e08r2n14:20092:23168 [0] NCCL INFO Connected all rings comm 0x2aeefc000ab0 nRanks 96 busId 4000
e08r3n10:20256:23242 [0] NCCL INFO Connected all rings comm 0x2b66fc000ab0 nRanks 96 busId 4000
e08r2n13:26096:29023 [3] NCCL INFO Connected all trees comm 0x2b8628000ab0 nRanks 96 busId 63000
e08r2n13:26096:29023 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2340:5287 [0] NCCL INFO Connected all rings comm 0x2b795c000ab0 nRanks 96 busId 4000
e08r2n13:26096:29023 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n09:28731:31688 [3] NCCL INFO Connected all trees comm 0x2b6a54000ab0 nRanks 96 busId 63000
e08r2n18:2634:5676 [0] NCCL INFO Connected all rings comm 0x2acbdc000ab0 nRanks 96 busId 4000
e08r2n09:28731:31688 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n09:28731:31688 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22824:25586 [3] NCCL INFO Connected all rings comm 0x2ba5b0000ab0 nRanks 96 busId 63000
e08r2n17:22824:25586 [3] NCCL INFO Channel 00 : 39[63000] -> 38[43000] via P2P/IPC comm 0x2ba5b0000ab0 nRanks 96
e08r2n17:22824:25586 [3] NCCL INFO Channel 01 : 39[63000] -> 38[43000] via P2P/IPC comm 0x2ba5b0000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Connected all rings comm 0x2b6ff4000ab0 nRanks 96 busId 4000
e08r2n17:22824:25586 [3] NCCL INFO Connected all trees comm 0x2ba5b0000ab0 nRanks 96 busId 63000
e08r2n17:22824:25586 [3] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n17:22824:25586 [3] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31686:1949 [0] NCCL INFO Channel 01/0 : 64[4000] -> 68[4000] [send] via NET/IB/0 comm 0x2b0b28000ab0 nRanks 96
e08r2n08:11393:19499 [0] NCCL INFO Channel 01/0 : 0[4000] -> 4[4000] [send] via NET/IB/0 comm 0x2b0148000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 00/0 : 72[4000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab4d0000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 01/0 : 48[4000] -> 52[4000] [receive] via NET/IB/0 comm 0x2acf70000ab0 nRanks 96
e08r3n02:14034:16489 [0] NCCL INFO Channel 01/0 : 53[26000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b5a20000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 01/0 : 80[4000] -> 84[4000] [send] via NET/IB/0 comm 0x2ac910000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 00/0 : 40[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2b01e0000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 01/0 : 32[4000] -> 36[4000] [send] via NET/IB/0 comm 0x2af288000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 01/0 : 48[4000] -> 52[4000] [send] via NET/IB/0 comm 0x2af400000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 01/0 : 16[4000] -> 20[4000] [send] via NET/IB/0 comm 0x2b64c4000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 01/0 : 80[4000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae4c4000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 01/0 : 32[4000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b0528000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 00/0 : 56[4000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba6a8000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 00/0 : 24[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b24e8000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 01/0 : 0[4000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af480000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 01/0 : 21[26000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aeefc000ab0 nRanks 96
e08r3n11:31583:2060 [0] NCCL INFO Channel 00/0 : 88[4000] -> 92[4000] [receive] via NET/IB/0 comm 0x2af4fc000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 01/0 : 69[26000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b2a74000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 01/0 : 16[4000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b1c18000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 01/0 : 64[4000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7678000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 01/0 : 85[26000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b66fc000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 01/0 : 37[26000] -> 40[4000] [receive] via NET/IB/0 comm 0x2acbdc000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 01/0 : 5[26000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b795c000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 00/0 : 8[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6ff4000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 00/0 : 32[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2af288000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 00/0 : 48[4000] -> 56[4000] [send] via NET/IB/0 comm 0x2af400000ab0 nRanks 96
e08r2n08:11393:19499 [0] NCCL INFO Channel 00/0 : 64[4000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b0148000ab0 nRanks 96
e08r3n01:21694:24038 [1] NCCL INFO Channel 01/0 : 56[4000] -> 53[26000] [receive] via NET/IB/0 comm 0x2b9438000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 00/0 : 80[4000] -> 88[4000] [send] via NET/IB/0 comm 0x2ac910000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Channel 00/0 : 64[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b0b28000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 00/0 : 16[4000] -> 24[4000] [send] via NET/IB/0 comm 0x2b64c4000ab0 nRanks 96
e08r3n09:10074:13077 [1] NCCL INFO Channel 01/0 : 88[4000] -> 85[26000] [receive] via NET/IB/0 comm 0x2aef78000ab0 nRanks 96
e08r2n17:22822:25590 [1] NCCL INFO Channel 01/0 : 40[4000] -> 37[26000] [receive] via NET/IB/0 comm 0x2b7e78000ab0 nRanks 96
e08r2n13:26094:29027 [1] NCCL INFO Channel 01/0 : 24[4000] -> 21[26000] [receive] via NET/IB/0 comm 0x2b2f78000ab0 nRanks 96
e08r3n05:26317:29374 [1] NCCL INFO Channel 01/0 : 72[4000] -> 69[26000] [receive] via NET/IB/0 comm 0x2b7ef4000ab0 nRanks 96
e08r2n09:28729:31686 [1] NCCL INFO Channel 01/0 : 8[4000] -> 5[26000] [receive] via NET/IB/0 comm 0x2b70dc000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 00/0 : 52[4000] -> 57[26000] [send] via NET/IB/0 comm 0x2acf70000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 00/0 : 84[4000] -> 89[26000] [send] via NET/IB/0 comm 0x2ae4c4000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 00/0 : 36[4000] -> 41[26000] [send] via NET/IB/0 comm 0x2b0528000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 00/0 : 20[4000] -> 25[26000] [send] via NET/IB/0 comm 0x2b1c18000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 00/0 : 68[4000] -> 73[26000] [send] via NET/IB/0 comm 0x2b7678000ab0 nRanks 96
e08r3n02:14034:16489 [0] NCCL INFO Channel 00/0 : 56[4000] -> 60[4000] [send] via NET/IB/0 comm 0x2b5a20000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 00/0 : 4[4000] -> 9[26000] [send] via NET/IB/0 comm 0x2af480000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 00/0 : 24[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2aeefc000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 00/0 : 40[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2acbdc000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 00/0 : 8[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2b795c000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 00/0 : 72[4000] -> 76[4000] [send] via NET/IB/0 comm 0x2b2a74000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 00/0 : 88[4000] -> 92[4000] [send] via NET/IB/0 comm 0x2b66fc000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 01/0 : 45[26000] -> 52[4000] [receive] via NET/IB/0 comm 0x2acf70000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 01/0 : 36[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2b0528000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 01/0 : 77[26000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae4c4000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 01/0 : 68[4000] -> 76[4000] [send] via NET/IB/0 comm 0x2b7678000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 01/0 : 13[26000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b1c18000ab0 nRanks 96
e08r3n02:14034:16489 [0] NCCL INFO Channel 00/0 : 48[4000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b5a20000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 01/0 : 4[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2af480000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 00/0 : 16[4000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aeefc000ab0 nRanks 96
e08r3n02:14036:16488 [1] NCCL INFO Channel 00/0 : 57[26000] -> 52[4000] [send] via NET/IB/0 comm 0x2b0654000ab0 nRanks 96
e08r2n14:20094:23169 [1] NCCL INFO Channel 00/0 : 25[26000] -> 20[4000] [send] via NET/IB/0 comm 0x2b5774000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 00/0 : 8[4000] -> 17[26000] [send] via NET/IB/0 comm 0x2b795c000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 00/0 : 72[4000] -> 81[26000] [send] via NET/IB/0 comm 0x2b2a74000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 00/0 : 40[4000] -> 49[26000] [send] via NET/IB/0 comm 0x2acbdc000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 00/0 : 80[4000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b66fc000ab0 nRanks 96
e08r2n10:2342:5288 [1] NCCL INFO Channel 00/0 : 9[26000] -> 4[4000] [send] via NET/IB/0 comm 0x2ac2d8000ab0 nRanks 96
e08r3n10:20258:23243 [1] NCCL INFO Channel 00/0 : 89[26000] -> 84[4000] [send] via NET/IB/0 comm 0x2b150c000ab0 nRanks 96
e08r2n18:2636:5675 [1] NCCL INFO Channel 00/0 : 41[26000] -> 36[4000] [send] via NET/IB/0 comm 0x2ac7e4000ab0 nRanks 96
e08r3n06:24083:27040 [1] NCCL INFO Channel 00/0 : 73[26000] -> 68[4000] [send] via NET/IB/0 comm 0x2ad250000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 01/0 : 44[4000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba6a8000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 01/0 : 12[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b24e8000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 01/0 : 4[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6ff4000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 01/0 : 36[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2b01e0000ab0 nRanks 96
e08r3n11:31583:2060 [0] NCCL INFO Channel 01/0 : 92[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2af4fc000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 01/0 : 68[4000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab4d0000ab0 nRanks 96
e08r3n07:20447:22839 [1] NCCL INFO Channel 01/0 : 84[4000] -> 77[26000] [receive] via NET/IB/0 comm 0x2b8f74000ab0 nRanks 96
e08r2n11:7502:10718 [1] NCCL INFO Channel 01/0 : 20[4000] -> 13[26000] [receive] via NET/IB/0 comm 0x2b7f44000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 00/0 : 32[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af400000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 00/0 : 16[4000] -> 33[26000] [send] via NET/IB/0 comm 0x2b64c4000ab0 nRanks 96
e08r2n19:10428:13638 [1] NCCL INFO Channel 01/0 : 52[4000] -> 45[26000] [receive] via NET/IB/0 comm 0x2b35b4000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 00/0 : 17[26000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b795c000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 00/0 : 81[26000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b2a74000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 00/0 : 64[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac910000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 00/0 : 49[26000] -> 40[4000] [receive] via NET/IB/0 comm 0x2acbdc000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 01/0 : 52[4000] -> 45[26000] [send] via NET/IB/0 comm 0x2acf70000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 01/0 : 84[4000] -> 77[26000] [send] via NET/IB/0 comm 0x2ae4c4000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 01/0 : 20[4000] -> 13[26000] [send] via NET/IB/0 comm 0x2b1c18000ab0 nRanks 96
e08r3n02:14034:16489 [0] NCCL INFO Channel 00/0 : 56[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2b5a20000ab0 nRanks 96
e08r3n08:14729:17767 [1] NCCL INFO Channel 00/0 : 81[26000] -> 72[4000] [send] via NET/IB/0 comm 0x2b1500000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 00/0 : 24[4000] -> 16[4000] [send] via NET/IB/0 comm 0x2aeefc000ab0 nRanks 96
e08r2n12:10509:13654 [1] NCCL INFO Channel 00/0 : 17[26000] -> 8[4000] [send] via NET/IB/0 comm 0x2baf28000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 01/0 : 12[4000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af480000ab0 nRanks 96
e08r3n00:29327:31836 [1] NCCL INFO Channel 00/0 : 49[26000] -> 40[4000] [send] via NET/IB/0 comm 0x2b1704000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 00/0 : 88[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2b66fc000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 01/0 : 44[4000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b0528000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 01/0 : 76[4000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7678000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 01/0 : 12[4000] -> 28[4000] [send] via NET/IB/0 comm 0x2b6ff4000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 01/0 : 44[4000] -> 60[4000] [send] via NET/IB/0 comm 0x2b01e0000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 01/0 : 61[26000] -> 76[4000] [receive] via NET/IB/0 comm 0x2ab4d0000ab0 nRanks 96
e08r2n12:10509:13654 [1] NCCL INFO Channel 00 : 17[26000] -> 16[4000] via P2P/IPC comm 0x2baf28000ab0 nRanks 96
e08r2n12:10509:13654 [1] NCCL INFO Channel 01 : 17[26000] -> 16[4000] via P2P/IPC comm 0x2baf28000ab0 nRanks 96
e08r3n08:14729:17767 [1] NCCL INFO Channel 00 : 81[26000] -> 80[4000] via P2P/IPC comm 0x2b1500000ab0 nRanks 96
e08r3n08:14729:17767 [1] NCCL INFO Channel 01 : 81[26000] -> 80[4000] via P2P/IPC comm 0x2b1500000ab0 nRanks 96
e08r3n00:29327:31836 [1] NCCL INFO Channel 00 : 49[26000] -> 48[4000] via P2P/IPC comm 0x2b1704000ab0 nRanks 96
e08r3n00:29327:31836 [1] NCCL INFO Channel 01 : 49[26000] -> 48[4000] via P2P/IPC comm 0x2b1704000ab0 nRanks 96
e08r2n12:10510:13650 [2] NCCL INFO Connected all trees comm 0x2b61d0000ab0 nRanks 96 busId 43000
e08r2n12:10510:13650 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10510:13650 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n08:14730:17765 [2] NCCL INFO Connected all trees comm 0x2af108000ab0 nRanks 96 busId 43000
e08r3n08:14730:17765 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14730:17765 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n07:20447:22839 [1] NCCL INFO Channel 00 : 77[26000] -> 76[4000] via P2P/IPC comm 0x2b8f74000ab0 nRanks 96
e08r3n07:20447:22839 [1] NCCL INFO Channel 01 : 77[26000] -> 76[4000] via P2P/IPC comm 0x2b8f74000ab0 nRanks 96
e08r2n11:7502:10718 [1] NCCL INFO Channel 00 : 13[26000] -> 12[4000] via P2P/IPC comm 0x2b7f44000ab0 nRanks 96
e08r2n11:7502:10718 [1] NCCL INFO Channel 01 : 13[26000] -> 12[4000] via P2P/IPC comm 0x2b7f44000ab0 nRanks 96
e08r3n00:29328:31837 [2] NCCL INFO Connected all trees comm 0x2ad7f0000ab0 nRanks 96 busId 43000
e08r3n00:29328:31837 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n00:29328:31837 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10428:13638 [1] NCCL INFO Channel 00 : 45[26000] -> 44[4000] via P2P/IPC comm 0x2b35b4000ab0 nRanks 96
e08r2n19:10428:13638 [1] NCCL INFO Channel 01 : 45[26000] -> 44[4000] via P2P/IPC comm 0x2b35b4000ab0 nRanks 96
e08r3n07:20448:22834 [2] NCCL INFO Connected all trees comm 0x2b46e4000ab0 nRanks 96 busId 43000
e08r3n07:20448:22834 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n07:20448:22834 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7503:10713 [2] NCCL INFO Connected all trees comm 0x2b520c000ab0 nRanks 96 busId 43000
e08r2n11:7503:10713 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n11:7503:10713 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10429:13634 [2] NCCL INFO Connected all trees comm 0x2b0404000ab0 nRanks 96 busId 43000
e08r2n19:10429:13634 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n19:10429:13634 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10508:13653 [0] NCCL INFO Channel 00/0 : 33[26000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b64c4000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 00/0 : 32[4000] -> 65[26000] [send] via NET/IB/0 comm 0x2af288000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 00/0 : 57[26000] -> 52[4000] [receive] via NET/IB/0 comm 0x2acf70000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Channel 00/0 : 64[4000] -> 0[4000] [send] via NET/IB/0 comm 0x2b0b28000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 00/0 : 89[26000] -> 84[4000] [receive] via NET/IB/0 comm 0x2ae4c4000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 00/0 : 25[26000] -> 20[4000] [receive] via NET/IB/0 comm 0x2b1c18000ab0 nRanks 96
e08r2n16:15465:18186 [1] NCCL INFO Channel 00/0 : 33[26000] -> 16[4000] [send] via NET/IB/0 comm 0x2ad418000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 00/0 : 80[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2ac910000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 00/0 : 48[4000] -> 32[4000] [send] via NET/IB/0 comm 0x2af400000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 01/0 : 28[4000] -> 12[4000] [receive] via NET/IB/0 comm 0x2b6ff4000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 00/0 : 12[4000] -> 8[4000] [receive] via NET/IB/0 comm 0x2b795c000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 00/0 : 76[4000] -> 72[4000] [receive] via NET/IB/0 comm 0x2b2a74000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 01/0 : 60[4000] -> 44[4000] [receive] via NET/IB/0 comm 0x2b01e0000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 00/0 : 44[4000] -> 40[4000] [receive] via NET/IB/0 comm 0x2acbdc000ab0 nRanks 96
e08r3n03:24231:27079 [1] NCCL INFO Channel 01/0 : 76[4000] -> 61[26000] [receive] via NET/IB/0 comm 0x2b32ac000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 01/0 : 92[4000] -> 28[4000] [receive] via NET/IB/0 comm 0x2b24e8000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 01/0 : 29[26000] -> 60[4000] [receive] via NET/IB/0 comm 0x2ba6a8000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 01/0 : 76[4000] -> 61[26000] [send] via NET/IB/0 comm 0x2ab4d0000ab0 nRanks 96
e08r2n16:15465:18186 [1] NCCL INFO Channel 00 : 33[26000] -> 32[4000] via P2P/IPC comm 0x2ad418000ab0 nRanks 96
e08r3n02:14036:16488 [1] NCCL INFO Channel 00 : 57[26000] -> 56[4000] via P2P/IPC comm 0x2b0654000ab0 nRanks 96
e08r2n16:15465:18186 [1] NCCL INFO Channel 01 : 33[26000] -> 32[4000] via P2P/IPC comm 0x2ad418000ab0 nRanks 96
e08r3n02:14036:16488 [1] NCCL INFO Channel 01 : 57[26000] -> 56[4000] via P2P/IPC comm 0x2b0654000ab0 nRanks 96
e08r3n10:20258:23243 [1] NCCL INFO Channel 00 : 89[26000] -> 88[4000] via P2P/IPC comm 0x2b150c000ab0 nRanks 96
e08r3n10:20258:23243 [1] NCCL INFO Channel 01 : 89[26000] -> 88[4000] via P2P/IPC comm 0x2b150c000ab0 nRanks 96
e08r2n14:20094:23169 [1] NCCL INFO Channel 00 : 25[26000] -> 24[4000] via P2P/IPC comm 0x2b5774000ab0 nRanks 96
e08r2n14:20094:23169 [1] NCCL INFO Channel 01 : 25[26000] -> 24[4000] via P2P/IPC comm 0x2b5774000ab0 nRanks 96
e08r2n16:15466:18183 [2] NCCL INFO Connected all trees comm 0x2bb254000ab0 nRanks 96 busId 43000
e08r2n16:15466:18183 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n16:15466:18183 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14037:16485 [2] NCCL INFO Connected all trees comm 0x2aef3c000ab0 nRanks 96 busId 43000
e08r3n02:14037:16485 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n02:14037:16485 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n10:20259:23240 [2] NCCL INFO Connected all trees comm 0x2b85ec000ab0 nRanks 96 busId 43000
e08r3n10:20259:23240 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n10:20259:23240 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20095:23164 [2] NCCL INFO Connected all trees comm 0x2baeac000ab0 nRanks 96 busId 43000
e08r2n14:20095:23164 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n14:20095:23164 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n03:24231:27079 [1] NCCL INFO Channel 00 : 61[26000] -> 60[4000] via P2P/IPC comm 0x2b32ac000ab0 nRanks 96
e08r3n03:24231:27079 [1] NCCL INFO Channel 01 : 61[26000] -> 60[4000] via P2P/IPC comm 0x2b32ac000ab0 nRanks 96
e08r3n03:24232:27074 [2] NCCL INFO Connected all trees comm 0x2acfe4000ab0 nRanks 96 busId 43000
e08r3n03:24232:27074 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n03:24232:27074 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31686:1949 [0] NCCL INFO Channel 00/0 : 0[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b0b28000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 00/0 : 65[26000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af288000ab0 nRanks 96
e08r3n04:31688:1950 [1] NCCL INFO Channel 00/0 : 65[26000] -> 32[4000] [send] via NET/IB/0 comm 0x2af494000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Channel 01/0 : 52[4000] -> 48[4000] [send] via NET/IB/0 comm 0x2acf70000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 00/0 : 24[4000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b64c4000ab0 nRanks 96
e08r3n09:10072:13076 [0] NCCL INFO Channel 01/0 : 84[4000] -> 80[4000] [send] via NET/IB/0 comm 0x2ae4c4000ab0 nRanks 96
e08r2n08:11393:19499 [0] NCCL INFO Channel 00/0 : 0[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b0148000ab0 nRanks 96
e08r2n13:26093:29026 [0] NCCL INFO Channel 01/0 : 20[4000] -> 16[4000] [send] via NET/IB/0 comm 0x2b1c18000ab0 nRanks 96
e08r2n15:23879:26734 [1] NCCL INFO Channel 01/0 : 60[4000] -> 29[26000] [receive] via NET/IB/0 comm 0x2b0b84000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 01/0 : 76[4000] -> 68[4000] [send] via NET/IB/0 comm 0x2ab4d0000ab0 nRanks 96
e08r3n11:31583:2060 [0] NCCL INFO Channel 01/0 : 28[4000] -> 92[4000] [receive] via NET/IB/0 comm 0x2af4fc000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 01/0 : 28[4000] -> 92[4000] [send] via NET/IB/0 comm 0x2b24e8000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 01/0 : 60[4000] -> 29[26000] [send] via NET/IB/0 comm 0x2ba6a8000ab0 nRanks 96
e08r3n04:31688:1950 [1] NCCL INFO Channel 00 : 65[26000] -> 64[4000] via P2P/IPC comm 0x2af494000ab0 nRanks 96
e08r3n04:31688:1950 [1] NCCL INFO Channel 01 : 65[26000] -> 64[4000] via P2P/IPC comm 0x2af494000ab0 nRanks 96
e08r3n04:31689:1947 [2] NCCL INFO Connected all trees comm 0x2ad6c0000ab0 nRanks 96 busId 43000
e08r3n04:31689:1947 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n04:31689:1947 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n15:23879:26734 [1] NCCL INFO Channel 00 : 29[26000] -> 28[4000] via P2P/IPC comm 0x2b0b84000ab0 nRanks 96
e08r2n15:23879:26734 [1] NCCL INFO Channel 01 : 29[26000] -> 28[4000] via P2P/IPC comm 0x2b0b84000ab0 nRanks 96
e08r2n15:23880:26728 [2] NCCL INFO Connected all trees comm 0x2b2058000ab0 nRanks 96 busId 43000
e08r2n15:23880:26728 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23880:26728 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20092:23168 [0] NCCL INFO Channel 00/0 : 28[4000] -> 24[4000] [receive] via NET/IB/0 comm 0x2aeefc000ab0 nRanks 96
e08r2n08:11393:19499 [0] NCCL INFO Channel 01/0 : 4[4000] -> 0[4000] [receive] via NET/IB/0 comm 0x2b0148000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 00/0 : 48[4000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af288000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Channel 00/0 : 80[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b0b28000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Channel 01/0 : 20[4000] -> 16[4000] [receive] via NET/IB/0 comm 0x2b64c4000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Channel 00/0 : 76[4000] -> 72[4000] [send] via NET/IB/0 comm 0x2ab4d0000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 01/0 : 28[4000] -> 12[4000] [send] via NET/IB/0 comm 0x2b24e8000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 01/0 : 60[4000] -> 44[4000] [send] via NET/IB/0 comm 0x2ba6a8000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 00/0 : 73[26000] -> 68[4000] [receive] via NET/IB/0 comm 0x2b7678000ab0 nRanks 96
e08r3n11:31583:2060 [0] NCCL INFO Channel 00/0 : 92[4000] -> 88[4000] [send] via NET/IB/0 comm 0x2af4fc000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO Connected all trees comm 0x2ab4d0000ab0 nRanks 96 busId 4000
e08r3n07:20445:22838 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n07:20445:22838 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n07:20447:22839 [1] NCCL INFO Connected all trees comm 0x2b8f74000ab0 nRanks 96 busId 26000
e08r3n07:20447:22839 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n07:20447:22839 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n06:24083:27040 [1] NCCL INFO Channel 00 : 73[26000] -> 72[4000] via P2P/IPC comm 0x2ad250000ab0 nRanks 96
e08r3n06:24083:27040 [1] NCCL INFO Channel 01 : 73[26000] -> 72[4000] via P2P/IPC comm 0x2ad250000ab0 nRanks 96
e08r2n12:10508:13653 [0] NCCL INFO Connected all trees comm 0x2b64c4000ab0 nRanks 96 busId 4000
e08r2n12:10508:13653 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10508:13653 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10509:13654 [1] NCCL INFO Connected all trees comm 0x2baf28000ab0 nRanks 96 busId 26000
e08r2n12:10509:13654 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n12:10509:13654 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n06:24084:27037 [2] NCCL INFO Connected all trees comm 0x2b7b74000ab0 nRanks 96 busId 43000
e08r3n06:24084:27037 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n06:24084:27037 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n00:29325:31835 [0] NCCL INFO Channel 00/0 : 56[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af400000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 00/0 : 88[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac910000ab0 nRanks 96
e08r2n15:23877:26732 [0] NCCL INFO Channel 00/0 : 28[4000] -> 24[4000] [send] via NET/IB/0 comm 0x2b24e8000ab0 nRanks 96
e08r2n16:15463:18187 [0] NCCL INFO Channel 01/0 : 36[4000] -> 32[4000] [receive] via NET/IB/0 comm 0x2af288000ab0 nRanks 96
e08r3n03:24229:27078 [0] NCCL INFO Channel 00/0 : 60[4000] -> 56[4000] [send] via NET/IB/0 comm 0x2ba6a8000ab0 nRanks 96
e08r3n04:31686:1949 [0] NCCL INFO Channel 01/0 : 68[4000] -> 64[4000] [receive] via NET/IB/0 comm 0x2b0b28000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Channel 01/0 : 72[4000] -> 69[26000] [send] via NET/IB/0 comm 0x2b2a74000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 01/0 : 12[4000] -> 4[4000] [send] via NET/IB/0 comm 0x2b6ff4000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 01/0 : 44[4000] -> 36[4000] [send] via NET/IB/0 comm 0x2b01e0000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Channel 01/0 : 68[4000] -> 64[4000] [send] via NET/IB/0 comm 0x2b7678000ab0 nRanks 96
e08r3n06:24081:27041 [0] NCCL INFO Connected all trees comm 0x2b2a74000ab0 nRanks 96 busId 4000
e08r2n15:23879:26734 [1] NCCL INFO Connected all trees comm 0x2b0b84000ab0 nRanks 96 busId 26000
e08r3n06:24081:27041 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23879:26734 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23879:26734 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n06:24081:27041 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n15:23877:26732 [0] NCCL INFO Connected all trees comm 0x2b24e8000ab0 nRanks 96 busId 4000
e08r2n15:23877:26732 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n15:23877:26732 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n06:24083:27040 [1] NCCL INFO Connected all trees comm 0x2ad250000ab0 nRanks 96 busId 26000
e08r3n06:24083:27040 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n06:24083:27040 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26317:29374 [1] NCCL INFO Channel 00 : 69[26000] -> 68[4000] via P2P/IPC comm 0x2b7ef4000ab0 nRanks 96
e08r3n05:26317:29374 [1] NCCL INFO Channel 01 : 69[26000] -> 68[4000] via P2P/IPC comm 0x2b7ef4000ab0 nRanks 96
e08r3n05:26315:29373 [0] NCCL INFO Connected all trees comm 0x2b7678000ab0 nRanks 96 busId 4000
e08r3n05:26315:29373 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n05:26315:29373 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26318:29369 [2] NCCL INFO Connected all trees comm 0x2b6538000ab0 nRanks 96 busId 43000
e08r3n05:26318:29369 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n05:26318:29369 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n05:26317:29374 [1] NCCL INFO Connected all trees comm 0x2b7ef4000ab0 nRanks 96 busId 26000
e08r3n05:26317:29374 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n05:26317:29374 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31688:1950 [1] NCCL INFO Connected all trees comm 0x2af494000ab0 nRanks 96 busId 26000
e08r3n04:31686:1949 [0] NCCL INFO Connected all trees comm 0x2b0b28000ab0 nRanks 96 busId 4000
e08r3n04:31688:1950 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n04:31686:1949 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n04:31686:1949 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n04:31688:1950 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14034:16489 [0] NCCL INFO Channel 00/0 : 60[4000] -> 56[4000] [receive] via NET/IB/0 comm 0x2b5a20000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 00/0 : 92[4000] -> 88[4000] [receive] via NET/IB/0 comm 0x2b66fc000ab0 nRanks 96
e08r3n00:29325:31835 [0] NCCL INFO Channel 01/0 : 52[4000] -> 48[4000] [receive] via NET/IB/0 comm 0x2af400000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Channel 01/0 : 84[4000] -> 80[4000] [receive] via NET/IB/0 comm 0x2ac910000ab0 nRanks 96
e08r2n11:7501:10717 [0] NCCL INFO Channel 00/0 : 12[4000] -> 8[4000] [send] via NET/IB/0 comm 0x2b6ff4000ab0 nRanks 96
e08r2n19:10427:13639 [0] NCCL INFO Channel 00/0 : 44[4000] -> 40[4000] [send] via NET/IB/0 comm 0x2b01e0000ab0 nRanks 96
e08r2n14:20092:23168 [0] NCCL INFO Channel 01/0 : 24[4000] -> 21[26000] [send] via NET/IB/0 comm 0x2aeefc000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 00/0 : 9[26000] -> 4[4000] [receive] via NET/IB/0 comm 0x2af480000ab0 nRanks 96
e08r3n07:20445:22838 [0] NCCL INFO comm 0x2ab4d0000ab0 rank 76 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n17:22820:25591 [0] NCCL INFO Channel 00/0 : 41[26000] -> 36[4000] [receive] via NET/IB/0 comm 0x2b0528000ab0 nRanks 96
e08r2n12:10509:13654 [1] NCCL INFO comm 0x2baf28000ab0 rank 17 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n12:10511:13649 [3] NCCL INFO comm 0x2b408c000ab0 rank 19 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n07:20448:22834 [2] NCCL INFO comm 0x2b46e4000ab0 rank 78 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n07:20447:22839 [1] NCCL INFO comm 0x2b8f74000ab0 rank 77 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n07:20449:22835 [3] NCCL INFO comm 0x2b27d8000ab0 rank 79 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n12:10508:13653 [0] NCCL INFO comm 0x2b64c4000ab0 rank 16 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n03:24229:27078 [0] NCCL INFO Connected all trees comm 0x2ba6a8000ab0 nRanks 96 busId 4000
e08r3n03:24229:27078 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n03:24229:27078 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n03:24231:27079 [1] NCCL INFO Connected all trees comm 0x2b32ac000ab0 nRanks 96 busId 26000
e08r3n03:24231:27079 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n03:24231:27079 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n12:10510:13650 [2] NCCL INFO comm 0x2b61d0000ab0 rank 18 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n11:31583:2060 [0] NCCL INFO Connected all trees comm 0x2af4fc000ab0 nRanks 96 busId 4000
e08r3n11:31583:2060 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n11:31583:2060 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n11:31585:2059 [1] NCCL INFO Connected all trees comm 0x2aed10000ab0 nRanks 96 busId 26000
e08r3n11:31585:2059 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n11:31585:2059 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7501:10717 [0] NCCL INFO Connected all trees comm 0x2b6ff4000ab0 nRanks 96 busId 4000
e08r2n11:7501:10717 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n11:7501:10717 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n11:7502:10718 [1] NCCL INFO Connected all trees comm 0x2b7f44000ab0 nRanks 96 busId 26000
e08r2n11:7502:10718 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n11:7502:10718 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10427:13639 [0] NCCL INFO Connected all trees comm 0x2b01e0000ab0 nRanks 96 busId 4000
e08r2n19:10427:13639 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n19:10427:13639 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n19:10428:13638 [1] NCCL INFO Connected all trees comm 0x2b35b4000ab0 nRanks 96 busId 26000
e08r2n19:10428:13638 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n19:10428:13638 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20092:23168 [0] NCCL INFO Connected all trees comm 0x2aeefc000ab0 nRanks 96 busId 4000
e08r2n14:20092:23168 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n14:20092:23168 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n00:29325:31835 [0] NCCL INFO Connected all trees comm 0x2af400000ab0 nRanks 96 busId 4000
e08r3n00:29325:31835 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n00:29325:31835 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n14:20094:23169 [1] NCCL INFO Connected all trees comm 0x2b5774000ab0 nRanks 96 busId 26000
e08r3n00:29327:31836 [1] NCCL INFO Connected all trees comm 0x2b1704000ab0 nRanks 96 busId 26000
e08r2n14:20094:23169 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n00:29327:31836 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n14:20094:23169 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n00:29327:31836 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2342:5288 [1] NCCL INFO Channel 00 : 9[26000] -> 8[4000] via P2P/IPC comm 0x2ac2d8000ab0 nRanks 96
e08r2n10:2342:5288 [1] NCCL INFO Channel 01 : 9[26000] -> 8[4000] via P2P/IPC comm 0x2ac2d8000ab0 nRanks 96
e08r2n18:2636:5675 [1] NCCL INFO Channel 00 : 41[26000] -> 40[4000] via P2P/IPC comm 0x2ac7e4000ab0 nRanks 96
e08r2n18:2636:5675 [1] NCCL INFO Channel 01 : 41[26000] -> 40[4000] via P2P/IPC comm 0x2ac7e4000ab0 nRanks 96
e08r3n08:14727:17764 [0] NCCL INFO Connected all trees comm 0x2ac910000ab0 nRanks 96 busId 4000
e08r3n08:14727:17764 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14727:17764 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n08:14729:17767 [1] NCCL INFO Connected all trees comm 0x2b1500000ab0 nRanks 96 busId 26000
e08r3n08:14729:17767 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n08:14729:17767 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26094:29027 [1] NCCL INFO Channel 00 : 21[26000] -> 20[4000] via P2P/IPC comm 0x2b2f78000ab0 nRanks 96
e08r2n13:26094:29027 [1] NCCL INFO Channel 01 : 21[26000] -> 20[4000] via P2P/IPC comm 0x2b2f78000ab0 nRanks 96
e08r2n10:2343:5283 [2] NCCL INFO Connected all trees comm 0x2b1738000ab0 nRanks 96 busId 43000
e08r2n10:2343:5283 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2343:5283 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n18:2637:5674 [2] NCCL INFO Connected all trees comm 0x2b3950000ab0 nRanks 96 busId 43000
e08r2n18:2637:5674 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2637:5674 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26094:29027 [1] NCCL INFO Connected all trees comm 0x2b2f78000ab0 nRanks 96 busId 26000
e08r2n13:26095:29022 [2] NCCL INFO Connected all trees comm 0x2b1ff8000ab0 nRanks 96 busId 43000
e08r2n13:26094:29027 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26094:29027 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26093:29026 [0] NCCL INFO Connected all trees comm 0x2b1c18000ab0 nRanks 96 busId 4000
e08r2n13:26095:29022 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26095:29022 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n13:26093:29026 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n13:26093:29026 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14034:16489 [0] NCCL INFO Channel 01/0 : 56[4000] -> 53[26000] [send] via NET/IB/0 comm 0x2b5a20000ab0 nRanks 96
e08r3n10:20256:23242 [0] NCCL INFO Channel 01/0 : 88[4000] -> 85[26000] [send] via NET/IB/0 comm 0x2b66fc000ab0 nRanks 96
e08r2n10:2340:5287 [0] NCCL INFO Channel 01/0 : 8[4000] -> 5[26000] [send] via NET/IB/0 comm 0x2b795c000ab0 nRanks 96
e08r2n18:2634:5676 [0] NCCL INFO Channel 01/0 : 40[4000] -> 37[26000] [send] via NET/IB/0 comm 0x2acbdc000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Channel 01/0 : 4[4000] -> 0[4000] [send] via NET/IB/0 comm 0x2af480000ab0 nRanks 96
e08r2n17:22820:25591 [0] NCCL INFO Channel 01/0 : 36[4000] -> 32[4000] [send] via NET/IB/0 comm 0x2b0528000ab0 nRanks 96
e08r2n15:23880:26728 [2] NCCL INFO comm 0x2b2058000ab0 rank 30 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n06:24081:27041 [0] NCCL INFO comm 0x2b2a74000ab0 rank 72 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n06:24083:27040 [1] NCCL INFO comm 0x2ad250000ab0 rank 73 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n05:26317:29374 [1] NCCL INFO comm 0x2b7ef4000ab0 rank 69 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n05:26319:29370 [3] NCCL INFO comm 0x2ae434000ab0 rank 71 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n04:31689:1947 [2] NCCL INFO comm 0x2ad6c0000ab0 rank 66 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n04:31688:1950 [1] NCCL INFO comm 0x2af494000ab0 rank 65 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n04:31690:1945 [3] NCCL INFO comm 0x2ac844000ab0 rank 67 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n06:24084:27037 [2] NCCL INFO comm 0x2b7b74000ab0 rank 74 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n15:23877:26732 [0] NCCL INFO comm 0x2b24e8000ab0 rank 28 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n15:23879:26734 [1] NCCL INFO comm 0x2b0b84000ab0 rank 29 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n15:23881:26730 [3] NCCL INFO comm 0x2b8004000ab0 rank 31 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n06:24085:27036 [3] NCCL INFO comm 0x2b716c000ab0 rank 75 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n02:14036:16488 [1] NCCL INFO Connected all trees comm 0x2b0654000ab0 nRanks 96 busId 26000
e08r3n02:14034:16489 [0] NCCL INFO Connected all trees comm 0x2b5a20000ab0 nRanks 96 busId 4000
e08r3n02:14036:16488 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n02:14036:16488 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n02:14034:16489 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n02:14034:16489 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n01:21694:24038 [1] NCCL INFO Channel 00 : 53[26000] -> 52[4000] via P2P/IPC comm 0x2b9438000ab0 nRanks 96
e08r3n01:21694:24038 [1] NCCL INFO Channel 01 : 53[26000] -> 52[4000] via P2P/IPC comm 0x2b9438000ab0 nRanks 96
e08r3n05:26318:29369 [2] NCCL INFO comm 0x2b6538000ab0 rank 70 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n05:26315:29373 [0] NCCL INFO comm 0x2b7678000ab0 rank 68 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n10:20256:23242 [0] NCCL INFO Connected all trees comm 0x2b66fc000ab0 nRanks 96 busId 4000
e08r3n10:20256:23242 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n10:20256:23242 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n10:20258:23243 [1] NCCL INFO Connected all trees comm 0x2b150c000ab0 nRanks 96 busId 26000
e08r3n04:31686:1949 [0] NCCL INFO comm 0x2b0b28000ab0 rank 64 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n10:20258:23243 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n10:20258:23243 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10074:13077 [1] NCCL INFO Channel 00 : 85[26000] -> 84[4000] via P2P/IPC comm 0x2aef78000ab0 nRanks 96
e08r3n09:10074:13077 [1] NCCL INFO Channel 01 : 85[26000] -> 84[4000] via P2P/IPC comm 0x2aef78000ab0 nRanks 96
e08r3n01:21692:24039 [0] NCCL INFO Connected all trees comm 0x2acf70000ab0 nRanks 96 busId 4000
e08r3n01:21692:24039 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n01:21692:24039 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2340:5287 [0] NCCL INFO Connected all trees comm 0x2b795c000ab0 nRanks 96 busId 4000
e08r2n10:2342:5288 [1] NCCL INFO Connected all trees comm 0x2ac2d8000ab0 nRanks 96 busId 26000
e08r2n10:2340:5287 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2340:5287 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n10:2342:5288 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n10:2342:5288 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n01:21695:24041 [2] NCCL INFO Connected all trees comm 0x2ad7ac000ab0 nRanks 96 busId 43000
e08r3n01:21694:24038 [1] NCCL INFO Connected all trees comm 0x2b9438000ab0 nRanks 96 busId 26000
e08r3n01:21695:24041 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n01:21695:24041 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n01:21694:24038 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n01:21694:24038 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10072:13076 [0] NCCL INFO Connected all trees comm 0x2ae4c4000ab0 nRanks 96 busId 4000
e08r3n09:10072:13076 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n09:10072:13076 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10074:13077 [1] NCCL INFO Connected all trees comm 0x2aef78000ab0 nRanks 96 busId 26000
e08r3n09:10074:13077 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n09:10075:13072 [2] NCCL INFO Connected all trees comm 0x2ba030000ab0 nRanks 96 busId 43000
e08r3n09:10074:13077 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n09:10075:13072 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r3n09:10075:13072 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n18:2634:5676 [0] NCCL INFO Connected all trees comm 0x2acbdc000ab0 nRanks 96 busId 4000
e08r2n18:2634:5676 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2634:5676 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n09:28729:31686 [1] NCCL INFO Channel 00 : 5[26000] -> 4[4000] via P2P/IPC comm 0x2b70dc000ab0 nRanks 96
e08r2n09:28729:31686 [1] NCCL INFO Channel 01 : 5[26000] -> 4[4000] via P2P/IPC comm 0x2b70dc000ab0 nRanks 96
e08r2n18:2636:5675 [1] NCCL INFO Connected all trees comm 0x2ac7e4000ab0 nRanks 96 busId 26000
e08r2n18:2636:5675 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n18:2636:5675 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22822:25590 [1] NCCL INFO Channel 00 : 37[26000] -> 36[4000] via P2P/IPC comm 0x2b7e78000ab0 nRanks 96
e08r2n17:22822:25590 [1] NCCL INFO Channel 01 : 37[26000] -> 36[4000] via P2P/IPC comm 0x2b7e78000ab0 nRanks 96
e08r2n09:28728:31689 [0] NCCL INFO Connected all trees comm 0x2af480000ab0 nRanks 96 busId 4000
e08r2n09:28728:31689 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n09:28728:31689 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n09:28730:31687 [2] NCCL INFO Connected all trees comm 0x2b2864000ab0 nRanks 96 busId 43000
e08r2n09:28730:31687 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n09:28729:31686 [1] NCCL INFO Connected all trees comm 0x2b70dc000ab0 nRanks 96 busId 26000
e08r2n09:28730:31687 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n09:28729:31686 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n09:28729:31686 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22820:25591 [0] NCCL INFO Connected all trees comm 0x2b0528000ab0 nRanks 96 busId 4000
e08r2n17:22820:25591 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n17:22820:25591 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22823:25587 [2] NCCL INFO Connected all trees comm 0x2afbe4000ab0 nRanks 96 busId 43000
e08r2n17:22823:25587 [2] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n17:22823:25587 [2] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n17:22822:25590 [1] NCCL INFO Connected all trees comm 0x2b7e78000ab0 nRanks 96 busId 26000
e08r2n17:22822:25590 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n17:22822:25590 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15463:18187 [0] NCCL INFO Connected all trees comm 0x2af288000ab0 nRanks 96 busId 4000
e08r2n16:15463:18187 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n16:15463:18187 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n16:15465:18186 [1] NCCL INFO Connected all trees comm 0x2ad418000ab0 nRanks 96 busId 26000
e08r2n16:15465:18186 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n16:15465:18186 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n08:11393:19499 [0] NCCL INFO Connected all trees comm 0x2b0148000ab0 nRanks 96 busId 4000
e08r2n08:11393:19499 [0] NCCL INFO Using tuning table 0 with LL128 disabled
e08r2n08:11393:19499 [0] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11393:19499 [0] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r2n08:11395:19506 [1] NCCL INFO Connected all trees comm 0x2b23f8000ab0 nRanks 96 busId 26000
e08r2n08:11395:19506 [1] NCCL INFO threadThresholds 8/8/64 | 768/8/64 | 8/8/256
e08r2n08:11395:19506 [1] NCCL INFO 2 coll channels, 2 p2p channels, 1 p2p channels per peer
e08r3n03:24229:27078 [0] NCCL INFO comm 0x2ba6a8000ab0 rank 60 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n03:24232:27074 [2] NCCL INFO comm 0x2acfe4000ab0 rank 62 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n03:24231:27079 [1] NCCL INFO comm 0x2b32ac000ab0 rank 61 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n03:24233:27076 [3] NCCL INFO comm 0x2b694c000ab0 rank 63 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n11:31583:2060 [0] NCCL INFO comm 0x2af4fc000ab0 rank 92 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n19:10429:13634 [2] NCCL INFO comm 0x2b0404000ab0 rank 46 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n19:10427:13639 [0] NCCL INFO comm 0x2b01e0000ab0 rank 44 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n00:29327:31836 [1] NCCL INFO comm 0x2b1704000ab0 rank 49 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n08:14727:17764 [0] NCCL INFO comm 0x2ac910000ab0 rank 80 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n08:14729:17767 [1] NCCL INFO comm 0x2b1500000ab0 rank 81 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n08:14731:17766 [3] NCCL INFO comm 0x2b9c40000ab0 rank 83 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n11:31586:2056 [2] NCCL INFO comm 0x2b32c4000ab0 rank 94 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n11:31587:2055 [3] NCCL INFO comm 0x2b9048000ab0 rank 95 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n19:10428:13638 [1] NCCL INFO comm 0x2b35b4000ab0 rank 45 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n11:31585:2059 [1] NCCL INFO comm 0x2aed10000ab0 rank 93 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n14:20095:23164 [2] NCCL INFO comm 0x2baeac000ab0 rank 26 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n11:7503:10713 [2] NCCL INFO comm 0x2b520c000ab0 rank 14 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n11:7501:10717 [0] NCCL INFO comm 0x2b6ff4000ab0 rank 12 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n11:7502:10718 [1] NCCL INFO comm 0x2b7f44000ab0 rank 13 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n19:10430:13635 [3] NCCL INFO comm 0x2b6b98000ab0 rank 47 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n08:14730:17765 [2] NCCL INFO comm 0x2af108000ab0 rank 82 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n14:20094:23169 [1] NCCL INFO comm 0x2b5774000ab0 rank 25 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n00:29325:31835 [0] NCCL INFO comm 0x2af400000ab0 rank 48 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n14:20096:23166 [3] NCCL INFO comm 0x2b60ec000ab0 rank 27 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n14:20092:23168 [0] NCCL INFO comm 0x2aeefc000ab0 rank 24 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n00:29328:31837 [2] NCCL INFO comm 0x2ad7f0000ab0 rank 50 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n11:7504:10715 [3] NCCL INFO comm 0x2bac84000ab0 rank 15 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n00:29330:31833 [3] NCCL INFO comm 0x2b0fd8000ab0 rank 51 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n13:26095:29022 [2] NCCL INFO comm 0x2b1ff8000ab0 rank 22 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n13:26096:29023 [3] NCCL INFO comm 0x2b8628000ab0 rank 23 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n13:26094:29027 [1] NCCL INFO comm 0x2b2f78000ab0 rank 21 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n13:26093:29026 [0] NCCL INFO comm 0x2b1c18000ab0 rank 20 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n02:14034:16489 [0] NCCL INFO comm 0x2b5a20000ab0 rank 56 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n02:14037:16485 [2] NCCL INFO comm 0x2aef3c000ab0 rank 58 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n01:21696:24040 [3] NCCL INFO comm 0x2b9e90000ab0 rank 55 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n10:2340:5287 [0] NCCL INFO comm 0x2b795c000ab0 rank 8 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n01:21692:24039 [0] NCCL INFO comm 0x2acf70000ab0 rank 52 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n01:21695:24041 [2] NCCL INFO comm 0x2ad7ac000ab0 rank 54 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n10:2342:5288 [1] NCCL INFO comm 0x2ac2d8000ab0 rank 9 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n10:2344:5284 [3] NCCL INFO comm 0x2b4c80000ab0 rank 11 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n09:10076:13074 [3] NCCL INFO comm 0x2ab9b8000ab0 rank 87 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n09:10074:13077 [1] NCCL INFO comm 0x2aef78000ab0 rank 85 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n18:2634:5676 [0] NCCL INFO comm 0x2acbdc000ab0 rank 40 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n18:2637:5674 [2] NCCL INFO comm 0x2b3950000ab0 rank 42 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n18:2638:5671 [3] NCCL INFO comm 0x2acc04000ab0 rank 43 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n09:10072:13076 [0] NCCL INFO comm 0x2ae4c4000ab0 rank 84 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n09:10075:13072 [2] NCCL INFO comm 0x2ba030000ab0 rank 86 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n18:2636:5675 [1] NCCL INFO comm 0x2ac7e4000ab0 rank 41 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n09:28731:31688 [3] NCCL INFO comm 0x2b6a54000ab0 rank 7 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n17:22823:25587 [2] NCCL INFO comm 0x2afbe4000ab0 rank 38 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n02:14038:16484 [3] NCCL INFO comm 0x2b2fa8000ab0 rank 59 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n02:14036:16488 [1] NCCL INFO comm 0x2b0654000ab0 rank 57 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n17:22820:25591 [0] NCCL INFO comm 0x2b0528000ab0 rank 36 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n10:20256:23242 [0] NCCL INFO comm 0x2b66fc000ab0 rank 88 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r3n10:20259:23240 [2] NCCL INFO comm 0x2b85ec000ab0 rank 90 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n08:11396:19502 [2] NCCL INFO comm 0x2b5920000ab0 rank 2 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n10:20260:23238 [3] NCCL INFO comm 0x2ad094000ab0 rank 91 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n16:15467:18182 [3] NCCL INFO comm 0x2b2028000ab0 rank 35 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n08:11393:19499 [0] NCCL INFO comm 0x2b0148000ab0 rank 0 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n16:15465:18186 [1] NCCL INFO comm 0x2ad418000ab0 rank 33 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n08:11397:19501 [3] NCCL INFO comm 0x2b6c88000ab0 rank 3 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n08:11395:19506 [1] NCCL INFO comm 0x2b23f8000ab0 rank 1 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n01:21694:24038 [1] NCCL INFO comm 0x2b9438000ab0 rank 53 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r3n10:20258:23243 [1] NCCL INFO comm 0x2b150c000ab0 rank 89 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n10:2343:5283 [2] NCCL INFO comm 0x2b1738000ab0 rank 10 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n17:22824:25586 [3] NCCL INFO comm 0x2ba5b0000ab0 rank 39 nranks 96 cudaDev 3 busId 63000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n09:28729:31686 [1] NCCL INFO comm 0x2b70dc000ab0 rank 5 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n09:28730:31687 [2] NCCL INFO comm 0x2b2864000ab0 rank 6 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n09:28728:31689 [0] NCCL INFO comm 0x2af480000ab0 rank 4 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
e08r2n17:22822:25590 [1] NCCL INFO comm 0x2b7e78000ab0 rank 37 nranks 96 cudaDev 1 busId 26000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n16:15466:18183 [2] NCCL INFO comm 0x2bb254000ab0 rank 34 nranks 96 cudaDev 2 busId 43000 localSize 404 used 37885072 bytes - Init COMPLETE
e08r2n16:15463:18187 [0] NCCL INFO comm 0x2af288000ab0 rank 32 nranks 96 cudaDev 0 busId 4000 localSize 404 used 21107856 bytes - Init COMPLETE
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}
{'loss': 3.0074, 'learning_rate': 2e-05, 'epoch': 0.01}