# nThreads: 1 nGpus: 1 nRanks: 1 minBytes: 7618 maxBytes: 1073741824 step: 2(factor) warmupIters: 5 iters: 20 agg iters: 1 validation: 1 graph: 0 # # Using devices # Rank 0 Pid 5748 on master device 0 [0000:9f:00.0] BW200 # Rank 1 Pid 5749 on master device 1 [0000:56:00.0] BW200 # Rank 2 Pid 5750 on master device 2 [0000:5d:00.0] BW200 # Rank 3 Pid 5751 on master device 3 [0000:05:00.0] BW200 # Rank 4 Pid 5752 on master device 4 [0000:e8:00.0] BW200 # Rank 5 Pid 5753 on master device 5 [0000:c1:00.0] BW200 # Rank 6 Pid 5754 on master device 6 [0000:ca:00.0] BW200 # Rank 7 Pid 5755 on master device 7 [0000:b1:00.0] BW200 # Rank 8 Pid 915 on node1 device 0 [0000:9f:00.0] BW200 # Rank 9 Pid 916 on node1 device 1 [0000:56:00.0] BW200 # Rank 10 Pid 917 on node1 device 2 [0000:5d:00.0] BW200 # Rank 11 Pid 918 on node1 device 3 [0000:05:00.0] BW200 # Rank 12 Pid 919 on node1 device 4 [0000:e8:00.0] BW200 # Rank 13 Pid 920 on node1 device 5 [0000:c1:00.0] BW200 # Rank 14 Pid 921 on node1 device 6 [0000:ca:00.0] BW200 # Rank 15 Pid 922 on node1 device 7 [0000:b1:00.0] BW200 master:5748:5748 [0] NCCL INFO Bootstrap : Using p14p2:10.17.26.195<0> master:5748:5748 [0] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory master:5748:5748 [0] NCCL INFO NET/Plugin : No plugin found, using internal implementation master:5748:5748 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" master:5748:5748 [0] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5748:5748 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! master:5748:5748 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! master:5748:5748 [0] NCCL INFO ROCr version 1.1 master:5748:5748 [0] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 RCCL version 2.18.3+hip6.1 HEAD:037e9b3 node1:920:920 [5] NCCL INFO ROCr version 1.1 node1:920:920 [5] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:920:920 [5] NCCL INFO Bootstrap : Using p14p2:10.17.26.196<0> node1:916:916 [1] NCCL INFO ROCr version 1.1 node1:916:916 [1] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:920:920 [5] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory node1:920:920 [5] NCCL INFO NET/Plugin : No plugin found, using internal implementation node1:920:920 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" node1:920:920 [5] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 node1:920:920 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! node1:920:920 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! node1:916:916 [1] NCCL INFO Bootstrap : Using p14p2:10.17.26.196<0> node1:916:916 [1] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory node1:916:916 [1] NCCL INFO NET/Plugin : No plugin found, using internal implementation node1:916:916 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" node1:916:916 [1] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 node1:916:916 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! node1:916:916 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! master:5749:5749 [1] NCCL INFO ROCr version 1.1 master:5749:5749 [1] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 master:5753:5753 [5] NCCL INFO ROCr version 1.1 master:5753:5753 [5] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:922:922 [7] NCCL INFO ROCr version 1.1 node1:922:922 [7] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:922:922 [7] NCCL INFO Bootstrap : Using p14p2:10.17.26.196<0> node1:922:922 [7] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory node1:922:922 [7] NCCL INFO NET/Plugin : No plugin found, using internal implementation node1:922:922 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" node1:922:922 [7] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 node1:922:922 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! node1:922:922 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! node1:918:918 [3] NCCL INFO ROCr version 1.1 node1:918:918 [3] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 master:5749:5749 [1] NCCL INFO Bootstrap : Using p14p2:10.17.26.195<0> master:5753:5753 [5] NCCL INFO Bootstrap : Using p14p2:10.17.26.195<0> master:5749:5749 [1] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory master:5749:5749 [1] NCCL INFO NET/Plugin : No plugin found, using internal implementation master:5749:5749 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" master:5753:5753 [5] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory master:5753:5753 [5] NCCL INFO NET/Plugin : No plugin found, using internal implementation master:5749:5749 [1] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5753:5753 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" master:5749:5749 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! master:5749:5749 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! master:5753:5753 [5] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5753:5753 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! master:5753:5753 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! node1:915:915 [0] NCCL INFO ROCr version 1.1 node1:915:915 [0] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:919:919 [4] NCCL INFO ROCr version 1.1 node1:919:919 [4] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:921:921 [6] NCCL INFO ROCr version 1.1 node1:921:921 [6] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:917:917 [2] NCCL INFO ROCr version 1.1 node1:917:917 [2] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 master:5750:5750 [2] NCCL INFO ROCr version 1.1 master:5750:5750 [2] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 master:5755:5755 [7] NCCL INFO ROCr version 1.1 master:5755:5755 [7] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 master:5754:5754 [6] NCCL INFO ROCr version 1.1 master:5754:5754 [6] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:918:918 [3] NCCL INFO Bootstrap : Using p14p2:10.17.26.196<0> node1:918:918 [3] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory node1:918:918 [3] NCCL INFO NET/Plugin : No plugin found, using internal implementation node1:918:918 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" node1:918:918 [3] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5752:5752 [4] NCCL INFO ROCr version 1.1 master:5752:5752 [4] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 node1:915:915 [0] NCCL INFO Bootstrap : Using p14p2:10.17.26.196<0> node1:915:915 [0] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory node1:915:915 [0] NCCL INFO NET/Plugin : No plugin found, using internal implementation node1:915:915 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" node1:915:915 [0] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 node1:919:919 [4] NCCL INFO Bootstrap : Using p14p2:10.17.26.196<0> node1:919:919 [4] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory node1:919:919 [4] NCCL INFO NET/Plugin : No plugin found, using internal implementation node1:919:919 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" node1:919:919 [4] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 node1:919:919 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! node1:919:919 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! node1:917:917 [2] NCCL INFO Bootstrap : Using p14p2:10.17.26.196<0> node1:917:917 [2] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory node1:917:917 [2] NCCL INFO NET/Plugin : No plugin found, using internal implementation node1:917:917 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" node1:917:917 [2] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 node1:917:917 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! node1:917:917 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! node1:921:921 [6] NCCL INFO Bootstrap : Using p14p2:10.17.26.196<0> node1:921:921 [6] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory node1:921:921 [6] NCCL INFO NET/Plugin : No plugin found, using internal implementation node1:921:921 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" node1:921:921 [6] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 node1:921:921 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! node1:921:921 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! node1:918:918 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! node1:918:918 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! node1:915:915 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! node1:915:915 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! master:5751:5751 [3] NCCL INFO ROCr version 1.1 master:5751:5751 [3] NCCL INFO Dmabuf feature disabled without NCCL_ENABLE_DMABUF_SUPPORT=1 master:5755:5755 [7] NCCL INFO Bootstrap : Using p14p2:10.17.26.195<0> master:5750:5750 [2] NCCL INFO Bootstrap : Using p14p2:10.17.26.195<0> master:5752:5752 [4] NCCL INFO Bootstrap : Using p14p2:10.17.26.195<0> master:5754:5754 [6] NCCL INFO Bootstrap : Using p14p2:10.17.26.195<0> master:5748:5789 [0] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 master:5751:5751 [3] NCCL INFO Bootstrap : Using p14p2:10.17.26.195<0> master:5750:5750 [2] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory master:5750:5750 [2] NCCL INFO NET/Plugin : No plugin found, using internal implementation master:5755:5755 [7] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory master:5755:5755 [7] NCCL INFO NET/Plugin : No plugin found, using internal implementation master:5750:5750 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" master:5750:5750 [2] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5755:5755 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" master:5755:5755 [7] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5750:5750 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! master:5750:5750 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! master:5755:5755 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! master:5755:5755 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! master:5754:5754 [6] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory master:5754:5754 [6] NCCL INFO NET/Plugin : No plugin found, using internal implementation master:5752:5752 [4] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory master:5752:5752 [4] NCCL INFO NET/Plugin : No plugin found, using internal implementation master:5751:5751 [3] NCCL INFO NET/Plugin : Plugin load (librccl-net.so) returned 2 : librccl-net.so: cannot open shared object file: No such file or directory master:5751:5751 [3] NCCL INFO NET/Plugin : No plugin found, using internal implementation master:5752:5752 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" master:5754:5754 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" master:5751:5751 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:121 NCCL WARN NUMA auto balancing enabled which can lead to variability in the RCCL performance! Disable by "sudo sysctl kernel.numa_balancing=0" master:5754:5754 [6] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5752:5752 [4] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5751:5751 [3] NCCL INFO Kernel version: 4.19.90-89.11.v2401.ky10.x86_64 master:5752:5752 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! master:5752:5752 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! master:5754:5754 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! master:5754:5754 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! master:5751:5751 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:142 NCCL WARN Missing "iommu=pt" from kernel command line which can lead to system instablity or hang! master:5751:5751 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:147 NCCL WARN Missing "HSA_FORCE_FINE_GRAIN_PCIE=1" from environment which can lead to low RCCL performance, system instablity or hang! node1:916:956 [1] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 node1:920:955 [5] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 node1:915:961 [0] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 node1:917:960 [2] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 node1:918:962 [3] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 node1:922:957 [7] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 node1:921:959 [6] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 node1:919:958 [4] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 master:5753:5791 [5] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 master:5749:5790 [1] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 master:5755:5793 [7] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 master:5751:5795 [3] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 master:5754:5794 [6] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 master:5752:5796 [4] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 master:5750:5792 [2] NCCL INFO NCCL_IB_HCA set to mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_7,mlx5_8,mlx5_9,mlx5_10 node1:916:956 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.196<0> node1:916:956 [1] NCCL INFO Using network IB node1:920:955 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.196<0> node1:920:955 [5] NCCL INFO Using network IB node1:922:957 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.196<0> node1:922:957 [7] NCCL INFO Using network IB node1:915:961 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.196<0> node1:915:961 [0] NCCL INFO Using network IB node1:919:958 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.196<0> node1:919:958 [4] NCCL INFO Using network IB node1:918:962 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.196<0> node1:918:962 [3] NCCL INFO Using network IB node1:921:959 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.196<0> node1:921:959 [6] NCCL INFO Using network IB node1:917:960 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.196<0> node1:917:960 [2] NCCL INFO Using network IB master:5748:5789 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.195<0> master:5748:5789 [0] NCCL INFO Using network IB master:5753:5791 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.195<0> master:5753:5791 [5] NCCL INFO Using network IB master:5751:5795 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.195<0> master:5751:5795 [3] NCCL INFO Using network IB master:5749:5790 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.195<0> master:5749:5790 [1] NCCL INFO Using network IB master:5755:5793 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.195<0> master:5755:5793 [7] NCCL INFO Using network IB master:5752:5796 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.195<0> master:5752:5796 [4] NCCL INFO Using network IB master:5750:5792 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.195<0> master:5750:5792 [2] NCCL INFO Using network IB master:5754:5794 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/IB [1]mlx5_2:1/IB [2]mlx5_3:1/IB [3]mlx5_4:1/IB [4]mlx5_7:1/IB [5]mlx5_8:1/IB [6]mlx5_9:1/IB [7]mlx5_10:1/IB [RO]; OOB p14p2:10.17.26.195<0> master:5754:5794 [6] NCCL INFO Using network IB master:5750:5792 [2] NCCL INFO comm 0x3d308b0 rank 2 nranks 16 cudaDev 2 busId 5d000 commId 0x2c23323128873473 - Init START master:5748:5789 [0] NCCL INFO comm 0x33b1de0 rank 0 nranks 16 cudaDev 0 busId 9f000 commId 0x2c23323128873473 - Init START master:5751:5795 [3] NCCL INFO comm 0x24c05f0 rank 3 nranks 16 cudaDev 3 busId 5000 commId 0x2c23323128873473 - Init START master:5749:5790 [1] NCCL INFO comm 0x257e160 rank 1 nranks 16 cudaDev 1 busId 56000 commId 0x2c23323128873473 - Init START master:5754:5794 [6] NCCL INFO comm 0x2fbfb50 rank 6 nranks 16 cudaDev 6 busId ca000 commId 0x2c23323128873473 - Init START master:5752:5796 [4] NCCL INFO comm 0x28cbfc0 rank 4 nranks 16 cudaDev 4 busId e8000 commId 0x2c23323128873473 - Init START master:5755:5793 [7] NCCL INFO comm 0x21dc490 rank 7 nranks 16 cudaDev 7 busId b1000 commId 0x2c23323128873473 - Init START node1:922:957 [7] NCCL INFO comm 0x38568d0 rank 15 nranks 16 cudaDev 7 busId b1000 commId 0x2c23323128873473 - Init START node1:920:955 [5] NCCL INFO comm 0x2e85470 rank 13 nranks 16 cudaDev 5 busId c1000 commId 0x2c23323128873473 - Init START master:5753:5791 [5] NCCL INFO comm 0x3e928f0 rank 5 nranks 16 cudaDev 5 busId c1000 commId 0x2c23323128873473 - Init START node1:921:959 [6] NCCL INFO comm 0x26c3520 rank 14 nranks 16 cudaDev 6 busId ca000 commId 0x2c23323128873473 - Init START master:5750:5792 [2] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:918:962 [3] NCCL INFO comm 0x3a34fa0 rank 11 nranks 16 cudaDev 3 busId 5000 commId 0x2c23323128873473 - Init START node1:915:961 [0] NCCL INFO comm 0x37c66a0 rank 8 nranks 16 cudaDev 0 busId 9f000 commId 0x2c23323128873473 - Init START master:5748:5789 [0] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:917:960 [2] NCCL INFO comm 0x23c8a20 rank 10 nranks 16 cudaDev 2 busId 5d000 commId 0x2c23323128873473 - Init START node1:919:958 [4] NCCL INFO comm 0x2b30900 rank 12 nranks 16 cudaDev 4 busId e8000 commId 0x2c23323128873473 - Init START node1:916:956 [1] NCCL INFO comm 0x264c650 rank 9 nranks 16 cudaDev 1 busId 56000 commId 0x2c23323128873473 - Init START node1:922:957 [7] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:920:955 [5] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 master:5752:5796 [4] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 master:5751:5795 [3] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 master:5755:5793 [7] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 master:5749:5790 [1] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 master:5753:5791 [5] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 master:5754:5794 [6] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:921:959 [6] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:918:962 [3] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:917:960 [2] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:919:958 [4] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:915:961 [0] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:916:956 [1] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:323 -> 2 node1:916:956 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:916:956 [1] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB node1:916:956 [1] NCCL INFO Setting affinity for GPU 1 to f0 node1:918:962 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:917:960 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:919:958 [4] NCCL INFO rocm_smi_lib: version 2.8.0.0 master:5755:5793 [7] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:915:961 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:922:957 [7] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:921:959 [6] NCCL INFO rocm_smi_lib: version 2.8.0.0 master:5754:5794 [6] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:920:955 [5] NCCL INFO rocm_smi_lib: version 2.8.0.0 master:5751:5795 [3] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:918:962 [3] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB node1:917:960 [2] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB node1:919:958 [4] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5750:5792 [2] NCCL INFO rocm_smi_lib: version 2.8.0.0 node1:915:961 [0] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB node1:922:957 [7] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB node1:921:959 [6] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB node1:920:955 [5] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB node1:920:955 [5] NCCL INFO Setting affinity for GPU 5 to f00000 master:5748:5789 [0] NCCL INFO rocm_smi_lib: version 2.8.0.0 master:5752:5796 [4] NCCL INFO rocm_smi_lib: version 2.8.0.0 master:5755:5793 [7] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5749:5790 [1] NCCL INFO rocm_smi_lib: version 2.8.0.0 master:5753:5791 [5] NCCL INFO rocm_smi_lib: version 2.8.0.0 master:5754:5794 [6] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5751:5795 [3] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5750:5792 [2] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5748:5789 [0] NCCL INFO NCCL_TOPO_DUMP_FILE set by environment to ./topo.xml master:5748:5789 [0] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5752:5796 [4] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5749:5790 [1] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5753:5791 [5] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB master:5749:5790 [1] NCCL INFO Setting affinity for GPU 1 to ffff0000,00000000,00000000,00000000,ffff0000 master:5753:5791 [5] NCCL INFO Setting affinity for GPU 5 to ffff0000,00000000,00000000,00000000,ffff0000,00000000,00000000 master:5752:5796 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 4 env param disable sdma copy node1:922:957 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 15 env param disable sdma copy master:5751:5795 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 3 env param disable sdma copy master:5751:5795 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 2/-1/-1->3->4 [2] 6/11/-1->3->-1 [3] 5/11/-1->3->-1 [4] 1/-1/-1->3->5 [5] 7/-1/-1->3->0 [6] 0/-1/-1->3->1 [7] 4/-1/-1->3->2 [8] 2/-1/-1->3->4 [9] 6/-1/-1->3->11 [10] 5/-1/-1->3->11 [11] 1/-1/-1->3->5 [12] 7/-1/-1->3->0 [13] 0/-1/-1->3->1 [14] 4/-1/-1->3->2 [15] 2/-1/-1->3->4 [16] 6/11/-1->3->-1 [17] 5/11/-1->3->-1 [18] 1/-1/-1->3->5 [19] 7/-1/-1->3->0 [20] 0/-1/-1->3->1 [21] 4/-1/-1->3->2 [22] 2/-1/-1->3->4 [23] 6/-1/-1->3->11 [24] 5/-1/-1->3->11 [25] 1/-1/-1->3->5 [26] 7/-1/-1->3->0 [27] 0/-1/-1->3->1 [28] 4/-1/-1->3->2 [29] 2/-1/-1->3->4 [30] 6/11/-1->3->-1 [31] 5/11/-1->3->-1 comm 0x24c05f0 nRanks 16 busId 5000 master:5752:5796 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 3/-1/-1->4->5 [2] 2/-1/-1->4->6 [3] 1/-1/-1->4->7 [4] 7/12/-1->4->-1 [5] 0/12/-1->4->-1 [6] 6/-1/-1->4->0 [7] 5/-1/-1->4->3 [8] 3/-1/-1->4->5 [9] 2/-1/-1->4->6 [10] 1/-1/-1->4->7 [11] 7/-1/-1->4->12 [12] 0/-1/-1->4->12 [13] 6/-1/-1->4->0 [14] 5/-1/-1->4->3 [15] 3/-1/-1->4->5 [16] 2/-1/-1->4->6 [17] 1/-1/-1->4->7 [18] 7/12/-1->4->-1 [19] 0/12/-1->4->-1 [20] 6/-1/-1->4->0 [21] 5/-1/-1->4->3 [22] 3/-1/-1->4->5 [23] 2/-1/-1->4->6 [24] 1/-1/-1->4->7 [25] 7/-1/-1->4->12 [26] 0/-1/-1->4->12 [27] 6/-1/-1->4->0 [28] 5/-1/-1->4->3 [29] 3/-1/-1->4->5 [30] 2/-1/-1->4->6 [31] 1/-1/-1->4->7 comm 0x28cbfc0 nRanks 16 busId e8000 master:5752:5796 [4] NCCL INFO -hcugon- create sdma group queue rank:4 localRanks:8 nRanks:16 sdma copy is not enabled master:5752:5796 [4] NCCL INFO P2P Chunksize set to 131072 node1:921:959 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 14 env param disable sdma copy node1:921:959 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 13/-1/-1->14->15 [2] 12/-1/-1->14->11 [3] -1/-1/-1->14->8 [4] 8/-1/-1->14->9 [5] 9/-1/-1->14->10 [6] 10/-1/-1->14->12 [7] 15/-1/-1->14->13 [8] 13/-1/-1->14->15 [9] 12/-1/-1->14->11 [10] -1/-1/-1->14->8 [11] 8/-1/-1->14->9 [12] 9/-1/-1->14->10 [13] 10/-1/-1->14->12 [14] 15/-1/-1->14->13 [15] 13/-1/-1->14->15 [16] 12/-1/-1->14->11 [17] -1/-1/-1->14->8 [18] 8/-1/-1->14->9 [19] 9/-1/-1->14->10 [20] 10/-1/-1->14->12 [21] 15/-1/-1->14->13 [22] 13/-1/-1->14->15 [23] 12/-1/-1->14->11 [24] -1/-1/-1->14->8 [25] 8/-1/-1->14->9 [26] 9/-1/-1->14->10 [27] 10/-1/-1->14->12 [28] 15/-1/-1->14->13 [29] 13/-1/-1->14->15 [30] 12/-1/-1->14->11 [31] -1/-1/-1->14->8 comm 0x26c3520 nRanks 16 busId ca000 node1:921:959 [6] NCCL INFO -hcugon- create sdma group queue rank:14 localRanks:8 nRanks:16 sdma copy is not enabled master:5754:5794 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 6 env param disable sdma copy master:5750:5792 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 2 env param disable sdma copy master:5750:5792 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 1/-1/-1->2->3 [2] 0/-1/-1->2->4 [3] 7/-1/-1->2->5 [4] -1/-1/-1->2->0 [5] 6/-1/-1->2->7 [6] 5/-1/-1->2->6 [7] 3/-1/-1->2->1 [8] 1/-1/-1->2->3 [9] 0/-1/-1->2->4 [10] 7/-1/-1->2->5 [11] -1/-1/-1->2->0 [12] 6/-1/-1->2->7 [13] 5/-1/-1->2->6 [14] 3/-1/-1->2->1 [15] 1/-1/-1->2->3 [16] 0/-1/-1->2->4 [17] 7/-1/-1->2->5 [18] -1/-1/-1->2->0 [19] 6/-1/-1->2->7 [20] 5/-1/-1->2->6 [21] 3/-1/-1->2->1 [22] 1/-1/-1->2->3 [23] 0/-1/-1->2->4 [24] 7/-1/-1->2->5 [25] -1/-1/-1->2->0 [26] 6/-1/-1->2->7 [27] 5/-1/-1->2->6 [28] 3/-1/-1->2->1 [29] 1/-1/-1->2->3 [30] 0/-1/-1->2->4 [31] 7/-1/-1->2->5 comm 0x3d308b0 nRanks 16 busId 5d000 master:5750:5792 [2] NCCL INFO -hcugon- create sdma group queue rank:2 localRanks:8 nRanks:16 sdma copy is not enabled master:5750:5792 [2] NCCL INFO P2P Chunksize set to 131072 master:5749:5790 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 1 env param disable sdma copy master:5749:5790 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] -1/-1/-1->1->2 [2] 7/-1/-1->1->5 [3] 0/-1/-1->1->4 [4] 6/-1/-1->1->3 [5] 5/-1/-1->1->6 [6] 3/-1/-1->1->7 [7] 2/-1/-1->1->0 [8] -1/-1/-1->1->2 [9] 7/-1/-1->1->5 [10] 0/-1/-1->1->4 [11] 6/-1/-1->1->3 [12] 5/-1/-1->1->6 [13] 3/-1/-1->1->7 [14] 2/-1/-1->1->0 [15] -1/-1/-1->1->2 [16] 7/-1/-1->1->5 [17] 0/-1/-1->1->4 [18] 6/-1/-1->1->3 [19] 5/-1/-1->1->6 [20] 3/-1/-1->1->7 [21] 2/-1/-1->1->0 [22] -1/-1/-1->1->2 [23] 7/-1/-1->1->5 [24] 0/-1/-1->1->4 [25] 6/-1/-1->1->3 [26] 5/-1/-1->1->6 [27] 3/-1/-1->1->7 [28] 2/-1/-1->1->0 [29] -1/-1/-1->1->2 [30] 7/-1/-1->1->5 [31] 0/-1/-1->1->4 comm 0x257e160 nRanks 16 busId 56000 master:5749:5790 [1] NCCL INFO -hcugon- create sdma group queue rank:1 localRanks:8 nRanks:16 sdma copy is not enabled master:5748:5789 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 0 env param disable sdma copy master:5755:5793 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 7 env param disable sdma copy master:5755:5793 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 6/-1/-1->7->0 [2] -1/-1/-1->7->1 [3] 4/-1/-1->7->2 [4] 5/-1/-1->7->4 [5] 2/-1/-1->7->3 [6] 1/15/-1->7->-1 [7] -1/-1/-1->7->6 [8] 6/-1/-1->7->0 [9] -1/-1/-1->7->1 [10] 4/-1/-1->7->2 [11] 5/-1/-1->7->4 [12] 2/-1/-1->7->3 [13] 1/-1/-1->7->15 [14] -1/-1/-1->7->6 [15] 6/-1/-1->7->0 [16] -1/-1/-1->7->1 [17] 4/-1/-1->7->2 [18] 5/-1/-1->7->4 [19] 2/-1/-1->7->3 [20] 1/15/-1->7->-1 [21] -1/-1/-1->7->6 [22] 6/-1/-1->7->0 [23] -1/-1/-1->7->1 [24] 4/-1/-1->7->2 [25] 5/-1/-1->7->4 [26] 2/-1/-1->7->3 [27] 1/-1/-1->7->15 [28] -1/-1/-1->7->6 [29] 6/-1/-1->7->0 [30] -1/-1/-1->7->1 [31] 4/-1/-1->7->2 comm 0x21dc490 nRanks 16 busId b1000 master:5755:5793 [7] NCCL INFO -hcugon- create sdma group queue rank:7 localRanks:8 nRanks:16 sdma copy is not enabled master:5755:5793 [7] NCCL INFO P2P Chunksize set to 131072 master:5753:5791 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 5 env param disable sdma copy master:5753:5791 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 4/-1/-1->5->6 [2] 1/-1/-1->5->0 [3] 2/-1/-1->5->3 [4] 3/-1/-1->5->7 [5] -1/-1/-1->5->1 [6] -1/-1/-1->5->2 [7] 6/-1/-1->5->4 [8] 4/-1/-1->5->6 [9] 1/-1/-1->5->0 [10] 2/-1/-1->5->3 [11] 3/-1/-1->5->7 [12] -1/-1/-1->5->1 [13] -1/-1/-1->5->2 [14] 6/-1/-1->5->4 [15] 4/-1/-1->5->6 [16] 1/-1/-1->5->0 [17] 2/-1/-1->5->3 [18] 3/-1/-1->5->7 [19] -1/-1/-1->5->1 [20] -1/-1/-1->5->2 [21] 6/-1/-1->5->4 [22] 4/-1/-1->5->6 [23] 1/-1/-1->5->0 [24] 2/-1/-1->5->3 [25] 3/-1/-1->5->7 [26] -1/-1/-1->5->1 [27] -1/-1/-1->5->2 [28] 6/-1/-1->5->4 [29] 4/-1/-1->5->6 [30] 1/-1/-1->5->0 [31] 2/-1/-1->5->3 comm 0x3e928f0 nRanks 16 busId c1000 master:5753:5791 [5] NCCL INFO -hcugon- create sdma group queue rank:5 localRanks:8 nRanks:16 sdma copy is not enabled master:5753:5791 [5] NCCL INFO P2P Chunksize set to 131072 master:5751:5795 [3] NCCL INFO -hcugon- create sdma group queue rank:3 localRanks:8 nRanks:16 sdma copy is not enabled master:5751:5795 [3] NCCL INFO P2P Chunksize set to 131072 master:5754:5794 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 5/-1/-1->6->7 [2] 4/-1/-1->6->3 [3] -1/-1/-1->6->0 [4] 0/-1/-1->6->1 [5] 1/-1/-1->6->2 [6] 2/-1/-1->6->4 [7] 7/-1/-1->6->5 [8] 5/-1/-1->6->7 [9] 4/-1/-1->6->3 [10] -1/-1/-1->6->0 [11] 0/-1/-1->6->1 [12] 1/-1/-1->6->2 [13] 2/-1/-1->6->4 [14] 7/-1/-1->6->5 [15] 5/-1/-1->6->7 [16] 4/-1/-1->6->3 [17] -1/-1/-1->6->0 [18] 0/-1/-1->6->1 [19] 1/-1/-1->6->2 [20] 2/-1/-1->6->4 [21] 7/-1/-1->6->5 [22] 5/-1/-1->6->7 [23] 4/-1/-1->6->3 [24] -1/-1/-1->6->0 [25] 0/-1/-1->6->1 [26] 1/-1/-1->6->2 [27] 2/-1/-1->6->4 [28] 7/-1/-1->6->5 [29] 5/-1/-1->6->7 [30] 4/-1/-1->6->3 [31] -1/-1/-1->6->0 comm 0x2fbfb50 nRanks 16 busId ca000 master:5754:5794 [6] NCCL INFO -hcugon- create sdma group queue rank:6 localRanks:8 nRanks:16 sdma copy is not enabled master:5754:5794 [6] NCCL INFO P2P Chunksize set to 131072 master:5749:5790 [1] NCCL INFO P2P Chunksize set to 131072 master:5748:5789 [0] NCCL INFO Channel 00/32 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 master:5748:5789 [0] NCCL INFO Channel 01/32 : 0 7 1 3 5 2 4 6 8 15 9 11 13 10 12 14 master:5748:5789 [0] NCCL INFO Channel 02/32 : 0 6 1 7 2 5 4 11 8 14 9 15 10 13 12 3 master:5748:5789 [0] NCCL INFO Channel 03/32 : 0 4 2 7 5 1 11 14 8 12 10 15 13 9 3 6 master:5748:5789 [0] NCCL INFO Channel 04/32 : 0 3 7 6 2 1 5 12 8 11 15 14 10 9 13 4 master:5748:5789 [0] NCCL INFO Channel 05/32 : 0 2 6 5 7 12 11 9 8 10 14 13 15 4 3 1 master:5748:5789 [0] NCCL INFO Channel 06/32 : 0 5 15 12 9 14 11 10 8 13 7 4 1 6 3 2 master:5748:5789 [0] NCCL INFO Channel 07/32 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 node1:920:955 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 13 env param disable sdma copy node1:920:955 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 12/-1/-1->13->14 [2] 9/-1/-1->13->8 [3] 10/-1/-1->13->11 [4] 11/-1/-1->13->15 [5] -1/-1/-1->13->9 [6] -1/-1/-1->13->10 [7] 14/-1/-1->13->12 [8] 12/-1/-1->13->14 [9] 9/-1/-1->13->8 [10] 10/-1/-1->13->11 [11] 11/-1/-1->13->15 [12] -1/-1/-1->13->9 [13] -1/-1/-1->13->10 [14] 14/-1/-1->13->12 [15] 12/-1/-1->13->14 [16] 9/-1/-1->13->8 [17] 10/-1/-1->13->11 [18] 11/-1/-1->13->15 [19] -1/-1/-1->13->9 [20] -1/-1/-1->13->10 [21] 14/-1/-1->13->12 [22] 12/-1/-1->13->14 [23] 9/-1/-1->13->8 [24] 10/-1/-1->13->11 [25] 11/-1/-1->13->15 [26] -1/-1/-1->13->9 [27] -1/-1/-1->13->10 [28] 14/-1/-1->13->12 [29] 12/-1/-1->13->14 [30] 9/-1/-1->13->8 [31] 10/-1/-1->13->11 comm 0x2e85470 nRanks 16 busId c1000 node1:920:955 [5] NCCL INFO -hcugon- create sdma group queue rank:13 localRanks:8 nRanks:16 sdma copy is not enabled node1:920:955 [5] NCCL INFO P2P Chunksize set to 131072 master:5748:5789 [0] NCCL INFO Channel 08/32 : 0 7 1 3 5 2 4 6 8 15 9 11 13 10 12 14 master:5748:5789 [0] NCCL INFO Channel 09/32 : 0 6 1 7 2 5 4 11 8 14 9 15 10 13 12 3 node1:918:962 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 11 env param disable sdma copy node1:918:962 [3] NCCL INFO Trees [0] 12/-1/-1->11->10 [1] 10/-1/-1->11->12 [2] 14/-1/-1->11->3 [3] 13/-1/-1->11->3 [4] 9/-1/-1->11->13 [5] 15/-1/-1->11->8 [6] 8/-1/-1->11->9 [7] 12/-1/-1->11->10 [8] 10/-1/-1->11->12 [9] 14/3/-1->11->-1 [10] 13/3/-1->11->-1 [11] 9/-1/-1->11->13 [12] 15/-1/-1->11->8 [13] 8/-1/-1->11->9 [14] 12/-1/-1->11->10 [15] 10/-1/-1->11->12 [16] 14/-1/-1->11->3 [17] 13/-1/-1->11->3 [18] 9/-1/-1->11->13 [19] 15/-1/-1->11->8 [20] 8/-1/-1->11->9 [21] 12/-1/-1->11->10 [22] 10/-1/-1->11->12 [23] 14/3/-1->11->-1 [24] 13/3/-1->11->-1 [25] 9/-1/-1->11->13 [26] 15/-1/-1->11->8 [27] 8/-1/-1->11->9 [28] 12/-1/-1->11->10 [29] 10/-1/-1->11->12 [30] 14/-1/-1->11->3 [31] 13/-1/-1->11->3 comm 0x3a34fa0 nRanks 16 busId 5000 node1:918:962 [3] NCCL INFO -hcugon- create sdma group queue rank:11 localRanks:8 nRanks:16 sdma copy is not enabled node1:918:962 [3] NCCL INFO P2P Chunksize set to 131072 node1:916:956 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 9 env param disable sdma copy node1:916:956 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] -1/-1/-1->9->10 [2] 15/-1/-1->9->13 [3] 8/-1/-1->9->12 [4] 14/-1/-1->9->11 [5] 13/-1/-1->9->14 [6] 11/-1/-1->9->15 [7] 10/-1/-1->9->8 [8] -1/-1/-1->9->10 [9] 15/-1/-1->9->13 [10] 8/-1/-1->9->12 [11] 14/-1/-1->9->11 [12] 13/-1/-1->9->14 [13] 11/-1/-1->9->15 [14] 10/-1/-1->9->8 [15] -1/-1/-1->9->10 [16] 15/-1/-1->9->13 [17] 8/-1/-1->9->12 [18] 14/-1/-1->9->11 [19] 13/-1/-1->9->14 [20] 11/-1/-1->9->15 [21] 10/-1/-1->9->8 [22] -1/-1/-1->9->10 [23] 15/-1/-1->9->13 [24] 8/-1/-1->9->12 [25] 14/-1/-1->9->11 [26] 13/-1/-1->9->14 [27] 11/-1/-1->9->15 [28] 10/-1/-1->9->8 [29] -1/-1/-1->9->10 [30] 15/-1/-1->9->13 [31] 8/-1/-1->9->12 comm 0x264c650 nRanks 16 busId 56000 node1:916:956 [1] NCCL INFO -hcugon- create sdma group queue rank:9 localRanks:8 nRanks:16 sdma copy is not enabled node1:916:956 [1] NCCL INFO P2P Chunksize set to 131072 node1:917:960 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 10 env param disable sdma copy node1:917:960 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 9/-1/-1->10->11 [2] 8/-1/-1->10->12 [3] 15/-1/-1->10->13 [4] -1/-1/-1->10->8 [5] 14/-1/-1->10->15 [6] 13/-1/-1->10->14 [7] 11/-1/-1->10->9 [8] 9/-1/-1->10->11 [9] 8/-1/-1->10->12 [10] 15/-1/-1->10->13 [11] -1/-1/-1->10->8 [12] 14/-1/-1->10->15 [13] 13/-1/-1->10->14 [14] 11/-1/-1->10->9 [15] 9/-1/-1->10->11 [16] 8/-1/-1->10->12 [17] 15/-1/-1->10->13 [18] -1/-1/-1->10->8 [19] 14/-1/-1->10->15 [20] 13/-1/-1->10->14 [21] 11/-1/-1->10->9 [22] 9/-1/-1->10->11 [23] 8/-1/-1->10->12 [24] 15/-1/-1->10->13 [25] -1/-1/-1->10->8 [26] 14/-1/-1->10->15 [27] 13/-1/-1->10->14 [28] 11/-1/-1->10->9 [29] 9/-1/-1->10->11 [30] 8/-1/-1->10->12 [31] 15/-1/-1->10->13 comm 0x23c8a20 nRanks 16 busId 5d000 node1:917:960 [2] NCCL INFO -hcugon- create sdma group queue rank:10 localRanks:8 nRanks:16 sdma copy is not enabled node1:917:960 [2] NCCL INFO P2P Chunksize set to 131072 node1:919:958 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 12 env param disable sdma copy node1:919:958 [4] NCCL INFO Trees [0] 13/-1/-1->12->11 [1] 11/-1/-1->12->13 [2] 10/-1/-1->12->14 [3] 9/-1/-1->12->15 [4] 15/-1/-1->12->4 [5] 8/-1/-1->12->4 [6] 14/-1/-1->12->8 [7] 13/-1/-1->12->11 [8] 11/-1/-1->12->13 [9] 10/-1/-1->12->14 [10] 9/-1/-1->12->15 [11] 15/4/-1->12->-1 [12] 8/4/-1->12->-1 [13] 14/-1/-1->12->8 [14] 13/-1/-1->12->11 [15] 11/-1/-1->12->13 [16] 10/-1/-1->12->14 [17] 9/-1/-1->12->15 [18] 15/-1/-1->12->4 [19] 8/-1/-1->12->4 [20] 14/-1/-1->12->8 [21] 13/-1/-1->12->11 [22] 11/-1/-1->12->13 [23] 10/-1/-1->12->14 [24] 9/-1/-1->12->15 [25] 15/4/-1->12->-1 [26] 8/4/-1->12->-1 [27] 14/-1/-1->12->8 [28] 13/-1/-1->12->11 [29] 11/-1/-1->12->13 [30] 10/-1/-1->12->14 [31] 9/-1/-1->12->15 comm 0x2b30900 nRanks 16 busId e8000 node1:919:958 [4] NCCL INFO -hcugon- create sdma group queue rank:12 localRanks:8 nRanks:16 sdma copy is not enabled node1:919:958 [4] NCCL INFO P2P Chunksize set to 131072 node1:922:957 [7] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 14/-1/-1->15->8 [2] -1/-1/-1->15->9 [3] 12/-1/-1->15->10 [4] 13/-1/-1->15->12 [5] 10/-1/-1->15->11 [6] 9/-1/-1->15->7 [7] -1/-1/-1->15->14 [8] 14/-1/-1->15->8 [9] -1/-1/-1->15->9 [10] 12/-1/-1->15->10 [11] 13/-1/-1->15->12 [12] 10/-1/-1->15->11 [13] 9/7/-1->15->-1 [14] -1/-1/-1->15->14 [15] 14/-1/-1->15->8 [16] -1/-1/-1->15->9 [17] 12/-1/-1->15->10 [18] 13/-1/-1->15->12 [19] 10/-1/-1->15->11 [20] 9/-1/-1->15->7 [21] -1/-1/-1->15->14 [22] 14/-1/-1->15->8 [23] -1/-1/-1->15->9 [24] 12/-1/-1->15->10 [25] 13/-1/-1->15->12 [26] 10/-1/-1->15->11 [27] 9/7/-1->15->-1 [28] -1/-1/-1->15->14 [29] 14/-1/-1->15->8 [30] -1/-1/-1->15->9 [31] 12/-1/-1->15->10 comm 0x38568d0 nRanks 16 busId b1000 node1:922:957 [7] NCCL INFO -hcugon- create sdma group queue rank:15 localRanks:8 nRanks:16 sdma copy is not enabled node1:922:957 [7] NCCL INFO P2P Chunksize set to 131072 node1:921:959 [6] NCCL INFO P2P Chunksize set to 131072 node1:915:961 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/init.cc:1312 NCCL WARN -hcugon- rank 8 env param disable sdma copy node1:915:961 [0] NCCL INFO Trees [0] 9/-1/-1->8->0 [1] 15/-1/-1->8->0 [2] 13/-1/-1->8->10 [3] 14/-1/-1->8->9 [4] 10/-1/-1->8->14 [5] 11/-1/-1->8->12 [6] 12/-1/-1->8->11 [7] 9/0/-1->8->-1 [8] 15/0/-1->8->-1 [9] 13/-1/-1->8->10 [10] 14/-1/-1->8->9 [11] 10/-1/-1->8->14 [12] 11/-1/-1->8->12 [13] 12/-1/-1->8->11 [14] 9/-1/-1->8->0 [15] 15/-1/-1->8->0 [16] 13/-1/-1->8->10 [17] 14/-1/-1->8->9 [18] 10/-1/-1->8->14 [19] 11/-1/-1->8->12 [20] 12/-1/-1->8->11 [21] 9/0/-1->8->-1 [22] 15/0/-1->8->-1 [23] 13/-1/-1->8->10 [24] 14/-1/-1->8->9 [25] 10/-1/-1->8->14 [26] 11/-1/-1->8->12 [27] 12/-1/-1->8->11 [28] 9/-1/-1->8->0 [29] 15/-1/-1->8->0 [30] 13/-1/-1->8->10 [31] 14/-1/-1->8->9 comm 0x37c66a0 nRanks 16 busId 9f000 node1:915:961 [0] NCCL INFO -hcugon- create sdma group queue rank:8 localRanks:8 nRanks:16 sdma copy is not enabled node1:915:961 [0] NCCL INFO P2P Chunksize set to 131072 master:5748:5789 [0] NCCL INFO Channel 10/32 : 0 4 2 7 5 1 11 14 8 12 10 15 13 9 3 6 master:5748:5789 [0] NCCL INFO Channel 11/32 : 0 3 7 6 2 1 5 12 8 11 15 14 10 9 13 4 master:5748:5789 [0] NCCL INFO Channel 12/32 : 0 2 6 5 7 12 11 9 8 10 14 13 15 4 3 1 master:5748:5789 [0] NCCL INFO Channel 13/32 : 0 5 15 12 9 14 11 10 8 13 7 4 1 6 3 2 master:5748:5789 [0] NCCL INFO Channel 14/32 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 master:5748:5789 [0] NCCL INFO Channel 15/32 : 0 7 1 3 5 2 4 6 8 15 9 11 13 10 12 14 master:5748:5789 [0] NCCL INFO Channel 16/32 : 0 6 1 7 2 5 4 11 8 14 9 15 10 13 12 3 master:5748:5789 [0] NCCL INFO Channel 17/32 : 0 4 2 7 5 1 11 14 8 12 10 15 13 9 3 6 master:5748:5789 [0] NCCL INFO Channel 18/32 : 0 3 7 6 2 1 5 12 8 11 15 14 10 9 13 4 master:5748:5789 [0] NCCL INFO Channel 19/32 : 0 2 6 5 7 12 11 9 8 10 14 13 15 4 3 1 master:5748:5789 [0] NCCL INFO Channel 20/32 : 0 5 15 12 9 14 11 10 8 13 7 4 1 6 3 2 master:5748:5789 [0] NCCL INFO Channel 21/32 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 master:5748:5789 [0] NCCL INFO Channel 22/32 : 0 7 1 3 5 2 4 6 8 15 9 11 13 10 12 14 master:5748:5789 [0] NCCL INFO Channel 23/32 : 0 6 1 7 2 5 4 11 8 14 9 15 10 13 12 3 master:5748:5789 [0] NCCL INFO Channel 24/32 : 0 4 2 7 5 1 11 14 8 12 10 15 13 9 3 6 master:5748:5789 [0] NCCL INFO Channel 25/32 : 0 3 7 6 2 1 5 12 8 11 15 14 10 9 13 4 master:5748:5789 [0] NCCL INFO Channel 26/32 : 0 2 6 5 7 12 11 9 8 10 14 13 15 4 3 1 master:5748:5789 [0] NCCL INFO Channel 27/32 : 0 5 15 12 9 14 11 10 8 13 7 4 1 6 3 2 master:5748:5789 [0] NCCL INFO Channel 28/32 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 master:5748:5789 [0] NCCL INFO Channel 29/32 : 0 7 1 3 5 2 4 6 8 15 9 11 13 10 12 14 master:5748:5789 [0] NCCL INFO Channel 30/32 : 0 6 1 7 2 5 4 11 8 14 9 15 10 13 12 3 master:5748:5789 [0] NCCL INFO Channel 31/32 : 0 4 2 7 5 1 11 14 8 12 10 15 13 9 3 6 master:5748:5789 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] 7/8/-1->0->-1 [2] 5/-1/-1->0->2 [3] 6/-1/-1->0->1 [4] 2/-1/-1->0->6 [5] 3/-1/-1->0->4 [6] 4/-1/-1->0->3 [7] 1/-1/-1->0->8 [8] 7/-1/-1->0->8 [9] 5/-1/-1->0->2 [10] 6/-1/-1->0->1 [11] 2/-1/-1->0->6 [12] 3/-1/-1->0->4 [13] 4/-1/-1->0->3 [14] 1/8/-1->0->-1 [15] 7/8/-1->0->-1 [16] 5/-1/-1->0->2 [17] 6/-1/-1->0->1 [18] 2/-1/-1->0->6 [19] 3/-1/-1->0->4 [20] 4/-1/-1->0->3 [21] 1/-1/-1->0->8 [22] 7/-1/-1->0->8 [23] 5/-1/-1->0->2 [24] 6/-1/-1->0->1 [25] 2/-1/-1->0->6 [26] 3/-1/-1->0->4 [27] 4/-1/-1->0->3 [28] 1/8/-1->0->-1 [29] 7/8/-1->0->-1 [30] 5/-1/-1->0->2 [31] 6/-1/-1->0->1 comm 0x33b1de0 nRanks 16 busId 9f000 master:5748:5789 [0] NCCL INFO -hcugon- create sdma group queue rank:0 localRanks:8 nRanks:16 sdma copy is not enabled master:5748:5789 [0] NCCL INFO P2P Chunksize set to 131072 node1:915:961 [0] NCCL INFO Channel 00/0 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 07/0 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 14/0 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 21/0 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 28/0 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 00/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 00/0 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 07/0 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 14/0 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 21/0 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 28/0 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 00/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 00/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 07/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 00/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 07/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 00/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 07/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 14/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:956 [1] NCCL INFO Channel 14/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 21/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 14/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 00/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 21/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 07/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 28/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 14/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:961 [0] NCCL INFO Channel 07/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 14/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 21/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 28/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:955 [5] NCCL INFO Channel 21/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 28/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 00/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 21/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:962 [3] NCCL INFO Channel 00/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 28/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 07/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:960 [2] NCCL INFO Channel 00/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:962 [3] NCCL INFO Channel 07/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 28/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 07/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:921:959 [6] NCCL INFO Channel 14/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 07/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 14/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 21/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 00/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 07/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:959 [6] NCCL INFO Channel 21/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 00/0 : 15[b1000] -> 0[9f000] [send] via NET/IB/0 comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 07/0 : 15[b1000] -> 0[9f000] [send] via NET/IB/0 comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 14/0 : 15[b1000] -> 0[9f000] [send] via NET/IB/0 comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 21/0 : 15[b1000] -> 0[9f000] [send] via NET/IB/0 comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 28/0 : 15[b1000] -> 0[9f000] [send] via NET/IB/0 comm 0x38568d0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 14/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:921:959 [6] NCCL INFO Channel 28/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:960 [2] NCCL INFO Channel 21/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 14/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 28/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 28/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 21/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 01/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 28/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 00/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 14/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 21/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 00/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 08/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 01/0 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 00/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 08/0 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 15/0 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 22/0 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 29/0 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 05/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 07/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 07/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 00/0 : 7[b1000] -> 8[9f000] [send] via NET/IB/0 comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 28/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:956 [1] NCCL INFO Channel 15/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 07/0 : 7[b1000] -> 8[9f000] [send] via NET/IB/0 comm 0x21dc490 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 07/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 14/0 : 7[b1000] -> 8[9f000] [send] via NET/IB/0 comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 21/0 : 7[b1000] -> 8[9f000] [send] via NET/IB/0 comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 28/0 : 7[b1000] -> 8[9f000] [send] via NET/IB/0 comm 0x21dc490 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 12/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 00/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 14/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 01/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 14/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 01/0 : 14[ca000] -> 0[9f000] [send] via NET/IB/1 comm 0x26c3520 nRanks 16 node1:915:961 [0] NCCL INFO Channel 01/0 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 08/0 : 14[ca000] -> 0[9f000] [send] via NET/IB/1 comm 0x26c3520 nRanks 16 node1:916:956 [1] NCCL INFO Channel 22/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 14/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 19/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 08/0 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 15/0 : 14[ca000] -> 0[9f000] [send] via NET/IB/1 comm 0x26c3520 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 21/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 15/0 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 22/0 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 29/0 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 05/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 07/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:959 [6] NCCL INFO Channel 22/0 : 14[ca000] -> 0[9f000] [send] via NET/IB/1 comm 0x26c3520 nRanks 16 node1:921:959 [6] NCCL INFO Channel 29/0 : 14[ca000] -> 0[9f000] [send] via NET/IB/1 comm 0x26c3520 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 21/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 08/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 26/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 21/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 28/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 12/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 29/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:960 [2] NCCL INFO Channel 15/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 28/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 14/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 28/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 19/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 22/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 21/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:961 [0] NCCL INFO Channel 26/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 01/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 29/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 28/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:962 [3] NCCL INFO Channel 01/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 08/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 15/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 01/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 08/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 08/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 15/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 05/0 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 22/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 29/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 01/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 01/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 22/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 29/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 15/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 22/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 12/0 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 19/0 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 01/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 08/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 08/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 29/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:956 [1] NCCL INFO Channel 04/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:955 [5] NCCL INFO Channel 26/0 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 02/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:956 [1] NCCL INFO Channel 11/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 15/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 15/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 08/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 09/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 22/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 22/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 18/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 15/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 05/0 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 29/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 16/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 29/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 22/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 01/0 : 6[ca000] -> 8[9f000] [send] via NET/IB/1 comm 0x2fbfb50 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 12/0 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 25/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 08/0 : 6[ca000] -> 8[9f000] [send] via NET/IB/1 comm 0x2fbfb50 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 15/0 : 6[ca000] -> 8[9f000] [send] via NET/IB/1 comm 0x2fbfb50 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 22/0 : 6[ca000] -> 8[9f000] [send] via NET/IB/1 comm 0x2fbfb50 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 29/0 : 6[ca000] -> 8[9f000] [send] via NET/IB/1 comm 0x2fbfb50 nRanks 16 node1:917:960 [2] NCCL INFO Channel 23/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 29/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 04/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 19/0 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 04/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 03/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 30/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 11/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 04/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 26/0 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 10/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 02/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 11/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 18/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 05/0 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 11/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 17/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 18/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 09/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 25/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 18/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 24/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 12/0 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:961 [0] NCCL INFO Channel 25/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 16/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 19/0 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:962 [3] NCCL INFO Channel 31/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 25/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 23/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 26/0 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:961 [0] NCCL INFO Channel 03/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 04/0 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 30/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 10/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 11/0 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 03/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 06/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 17/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 18/0 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 10/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 24/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 13/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:962 [3] NCCL INFO Channel 25/0 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 17/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 20/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 31/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 03/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 24/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 05/0 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 02/0 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 06/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 27/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:962 [3] NCCL INFO Channel 09/0 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 10/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:962 [3] NCCL INFO Channel 16/0 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 23/0 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 31/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 05/0 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 30/0 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 12/0 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 19/0 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 26/0 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 12/0 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 13/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 17/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 03/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 04/0 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 20/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 24/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 19/0 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 10/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 11/0 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 27/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 31/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:956 [1] NCCL INFO Channel 02/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 26/0 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 17/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 18/0 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 02/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 09/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 24/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 04/0 : 13[c1000] -> 4[e8000] [send] via NET/IB/4 comm 0x2e85470 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 25/0 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 06/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:961 [0] NCCL INFO Channel 09/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 05/0 : 15[b1000] -> 4[e8000] [send] via NET/IB/5 comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 12/0 : 15[b1000] -> 4[e8000] [send] via NET/IB/5 comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 19/0 : 15[b1000] -> 4[e8000] [send] via NET/IB/5 comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 11/0 : 13[c1000] -> 4[e8000] [send] via NET/IB/4 comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 18/0 : 13[c1000] -> 4[e8000] [send] via NET/IB/4 comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 25/0 : 13[c1000] -> 4[e8000] [send] via NET/IB/4 comm 0x2e85470 nRanks 16 node1:922:957 [7] NCCL INFO Channel 26/0 : 15[b1000] -> 4[e8000] [send] via NET/IB/5 comm 0x38568d0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 31/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 16/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 16/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 13/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 03/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 06/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 23/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 23/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 02/0 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 20/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 09/0 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 16/0 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 23/0 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 10/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 30/0 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 13/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 30/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 30/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 05/0 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 12/0 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 19/0 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 27/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 26/0 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 04/0 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 17/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 20/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 11/0 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 03/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:961 [0] NCCL INFO Channel 01/0 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 18/0 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 25/0 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 02/0 : 4[e8000] -> 11[5000] [send] via NET/IB/3 comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 09/0 : 4[e8000] -> 11[5000] [send] via NET/IB/3 comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 16/0 : 4[e8000] -> 11[5000] [send] via NET/IB/3 comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 23/0 : 4[e8000] -> 11[5000] [send] via NET/IB/3 comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 30/0 : 4[e8000] -> 11[5000] [send] via NET/IB/3 comm 0x28cbfc0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 27/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 24/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 08/0 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 04/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 06/0 : 13[c1000] -> 7[b1000] [send] via NET/IB/7 comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 10/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:955 [5] NCCL INFO Channel 13/0 : 13[c1000] -> 7[b1000] [send] via NET/IB/7 comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 20/0 : 13[c1000] -> 7[b1000] [send] via NET/IB/7 comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 03/0 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 27/0 : 13[c1000] -> 7[b1000] [send] via NET/IB/7 comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 10/0 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 15/0 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 31/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 17/0 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 24/0 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 31/0 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 17/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 11/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 22/0 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 24/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 05/0 : 7[b1000] -> 12[e8000] [send] via NET/IB/5 comm 0x21dc490 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 02/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 18/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 02/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 12/0 : 7[b1000] -> 12[e8000] [send] via NET/IB/5 comm 0x21dc490 nRanks 16 node1:915:961 [0] NCCL INFO Channel 29/0 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 19/0 : 7[b1000] -> 12[e8000] [send] via NET/IB/5 comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 31/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 26/0 : 7[b1000] -> 12[e8000] [send] via NET/IB/5 comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 04/0 : 5[c1000] -> 12[e8000] [send] via NET/IB/4 comm 0x3e928f0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 09/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 11/0 : 5[c1000] -> 12[e8000] [send] via NET/IB/4 comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 04/0 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 18/0 : 5[c1000] -> 12[e8000] [send] via NET/IB/4 comm 0x3e928f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 25/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 11/0 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 09/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 25/0 : 5[c1000] -> 12[e8000] [send] via NET/IB/4 comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 18/0 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 25/0 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 02/0 : 12[e8000] -> 3[5000] [send] via NET/IB/3 comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 09/0 : 12[e8000] -> 3[5000] [send] via NET/IB/3 comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 16/0 : 12[e8000] -> 3[5000] [send] via NET/IB/3 comm 0x2b30900 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 16/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 23/0 : 12[e8000] -> 3[5000] [send] via NET/IB/3 comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 30/0 : 12[e8000] -> 3[5000] [send] via NET/IB/3 comm 0x2b30900 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 06/0 : 5[c1000] -> 15[b1000] [send] via NET/IB/7 comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 16/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 04/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 03/0 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 13/0 : 5[c1000] -> 15[b1000] [send] via NET/IB/7 comm 0x3e928f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 10/0 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 20/0 : 5[c1000] -> 15[b1000] [send] via NET/IB/7 comm 0x3e928f0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 23/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 17/0 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 27/0 : 5[c1000] -> 15[b1000] [send] via NET/IB/7 comm 0x3e928f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 24/0 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 31/0 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 23/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 11/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 30/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 30/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 18/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 03/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 01/0 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 25/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 08/0 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 10/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 15/0 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 17/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 06/0 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 13/0 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 20/0 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 27/0 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 01/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 22/0 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 24/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 29/0 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 08/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 31/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:916:956 [1] NCCL INFO Channel 03/0 : 9[56000] -> 3[5000] [send] via NET/IB/2 comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 10/0 : 9[56000] -> 3[5000] [send] via NET/IB/2 comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 17/0 : 9[56000] -> 3[5000] [send] via NET/IB/2 comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 24/0 : 9[56000] -> 3[5000] [send] via NET/IB/2 comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 31/0 : 9[56000] -> 3[5000] [send] via NET/IB/2 comm 0x264c650 nRanks 16 node1:922:957 [7] NCCL INFO Channel 15/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 22/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 29/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 02/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 06/0 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 13/0 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 20/0 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 27/0 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 01/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 03/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 03/0 : 1[56000] -> 11[5000] [send] via NET/IB/2 comm 0x257e160 nRanks 16 node1:922:957 [7] NCCL INFO Channel 02/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 09/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 10/0 : 1[56000] -> 11[5000] [send] via NET/IB/2 comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 17/0 : 1[56000] -> 11[5000] [send] via NET/IB/2 comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 24/0 : 1[56000] -> 11[5000] [send] via NET/IB/2 comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 31/0 : 1[56000] -> 11[5000] [send] via NET/IB/2 comm 0x257e160 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 08/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 10/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 16/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 02/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 09/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 23/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 15/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 09/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 17/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 16/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 30/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 22/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 16/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 02/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 23/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 29/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 24/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 23/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 30/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 09/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 02/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 31/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 02/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 03/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 30/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 06/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 09/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 16/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:955 [5] NCCL INFO Channel 10/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 09/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 04/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 13/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 16/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 11/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:962 [3] NCCL INFO Channel 16/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 17/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 23/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 20/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 23/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:962 [3] NCCL INFO Channel 23/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 24/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 18/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 30/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:958 [4] NCCL INFO Channel 06/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:957 [7] NCCL INFO Channel 27/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 30/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:962 [3] NCCL INFO Channel 30/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 25/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:958 [4] NCCL INFO Channel 13/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 31/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 06/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 20/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 04/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 13/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 06/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:958 [4] NCCL INFO Channel 27/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 01/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 11/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 06/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 20/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 13/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 03/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 08/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 20/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 18/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 27/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 13/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 15/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:957 [7] NCCL INFO Channel 10/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 27/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 25/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 20/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 17/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 22/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 05/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 27/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 06/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 24/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 12/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 29/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:957 [7] NCCL INFO Channel 31/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 03/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 03/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 01/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 13/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:962 [3] NCCL INFO Channel 19/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 10/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 10/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 26/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 20/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 08/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 17/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 17/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 27/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:959 [6] NCCL INFO Channel 05/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 24/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 15/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 24/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:959 [6] NCCL INFO Channel 12/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 31/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 05/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 31/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 22/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 19/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:959 [6] NCCL INFO Channel 26/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 12/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 29/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 03/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 19/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 10/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 06/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 26/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 17/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 05/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:960 [2] NCCL INFO Channel 13/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 05/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 04/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 20/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:962 [3] NCCL INFO Channel 06/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 24/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 12/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 12/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 11/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 13/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 27/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:956 [1] NCCL INFO Channel 19/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 31/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 19/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 18/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 20/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 26/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 26/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:962 [3] NCCL INFO Channel 27/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 25/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 06/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 02/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 04/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 06/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 05/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 13/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 09/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 11/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 13/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 12/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 20/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 16/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 20/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 18/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 19/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 27/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 27/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 04/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:955 [5] NCCL INFO Channel 23/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:958 [4] NCCL INFO Channel 05/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 26/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 25/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:955 [5] NCCL INFO Channel 30/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 11/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:958 [4] NCCL INFO Channel 12/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 02/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 18/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:958 [4] NCCL INFO Channel 19/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:960 [2] NCCL INFO Channel 25/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 09/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 26/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 16/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 23/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 04/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 30/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 11/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 18/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 05/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 25/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 12/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 19/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 26/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:960 [2] NCCL INFO Connected all rings comm 0x23c8a20 nRanks 16 busId 5d000 master:5750:5792 [2] NCCL INFO Connected all rings comm 0x3d308b0 nRanks 16 busId 5d000 node1:917:960 [2] NCCL INFO Channel 01/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:960 [2] NCCL INFO Channel 08/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:960 [2] NCCL INFO Channel 15/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:960 [2] NCCL INFO Channel 22/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:960 [2] NCCL INFO Channel 29/0 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 01/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 08/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 15/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 22/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 29/0 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Connected all rings comm 0x33b1de0 nRanks 16 busId 9f000 master:5748:5789 [0] NCCL INFO Channel 03/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 10/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 17/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 24/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 31/0 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5795 [3] NCCL INFO Connected all rings comm 0x24c05f0 nRanks 16 busId 5000 node1:915:961 [0] NCCL INFO Connected all rings comm 0x37c66a0 nRanks 16 busId 9f000 node1:915:961 [0] NCCL INFO Channel 03/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 10/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 01/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 17/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 08/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 24/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 15/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 31/0 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 22/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 29/0 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:959 [6] NCCL INFO Connected all rings comm 0x26c3520 nRanks 16 busId ca000 node1:916:956 [1] NCCL INFO Connected all rings comm 0x264c650 nRanks 16 busId 56000 node1:916:956 [1] NCCL INFO Channel 01/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 08/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 15/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:959 [6] NCCL INFO Channel 01/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Connected all rings comm 0x2fbfb50 nRanks 16 busId ca000 node1:916:956 [1] NCCL INFO Channel 22/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:959 [6] NCCL INFO Channel 08/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:916:956 [1] NCCL INFO Channel 29/0 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:959 [6] NCCL INFO Channel 15/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:959 [6] NCCL INFO Channel 22/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:961 [0] NCCL INFO Channel 02/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 04/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:959 [6] NCCL INFO Channel 29/0 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:916:956 [1] NCCL INFO Channel 06/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 04/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 11/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 09/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 13/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 18/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5790 [1] NCCL INFO Connected all rings comm 0x257e160 nRanks 16 busId 56000 node1:916:956 [1] NCCL INFO Channel 20/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 25/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 27/0 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 11/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 16/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 18/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 23/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 25/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 30/0 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 01/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 01/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 08/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 08/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 15/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 15/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 22/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 22/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 29/0 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 29/0 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 04/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 06/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 11/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 13/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 18/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 02/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 20/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 25/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 04/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 27/0 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 09/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 11/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 16/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 18/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 02/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 23/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 09/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:962 [3] NCCL INFO Connected all rings comm 0x3a34fa0 nRanks 16 busId 5000 master:5748:5789 [0] NCCL INFO Channel 25/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 16/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 30/0 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 23/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 30/0 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 05/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 06/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 12/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 01/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 13/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 08/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 19/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 15/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 20/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 22/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 26/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 29/0 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 27/0 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 02/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:960 [2] NCCL INFO Channel 09/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:960 [2] NCCL INFO Channel 16/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:960 [2] NCCL INFO Channel 23/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:960 [2] NCCL INFO Channel 30/0 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:961 [0] NCCL INFO Channel 05/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 06/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 12/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 13/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 19/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 20/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 26/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 27/0 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5796 [4] NCCL INFO Connected all rings comm 0x28cbfc0 nRanks 16 busId e8000 master:5752:5796 [4] NCCL INFO Channel 01/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 08/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 15/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 22/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 29/0 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 03/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 04/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 10/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 11/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 17/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 18/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 24/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 25/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 31/0 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 03/0 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 10/0 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 17/0 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 24/0 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 31/0 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Connected all rings comm 0x2b30900 nRanks 16 busId e8000 node1:919:958 [4] NCCL INFO Channel 01/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 08/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 15/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 22/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 29/0 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 03/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 04/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 10/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:957 [7] NCCL INFO Connected all rings comm 0x38568d0 nRanks 16 busId b1000 node1:918:962 [3] NCCL INFO Channel 11/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 17/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 18/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 24/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 25/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 31/0 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 03/0 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 10/0 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 17/0 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 24/0 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 31/0 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5791 [5] NCCL INFO Connected all rings comm 0x3e928f0 nRanks 16 busId c1000 master:5753:5791 [5] NCCL INFO Channel 01/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 08/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 15/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 22/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 29/0 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 02/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 04/0 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 11/0 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 18/0 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 25/0 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 06/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 09/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 13/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 16/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 20/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 23/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 27/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 30/0 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 03/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 06/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 10/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 02/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 13/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 09/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 17/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 16/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 20/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 23/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 24/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 30/0 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 27/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 05/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 31/0 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 06/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 12/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 13/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 19/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 20/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 26/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 27/0 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5755:5793 [7] NCCL INFO Connected all rings comm 0x21dc490 nRanks 16 busId b1000 master:5750:5792 [2] NCCL INFO Channel 06/0 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 13/0 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 20/0 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 27/0 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 05/0 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 12/0 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 19/0 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 26/0 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 05/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:920:955 [5] NCCL INFO Connected all rings comm 0x2e85470 nRanks 16 busId c1000 master:5750:5792 [2] NCCL INFO Channel 12/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 19/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 03/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 26/0 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 04/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 10/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 11/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 17/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 18/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 24/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 25/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 31/0 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 02/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 05/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:955 [5] NCCL INFO Channel 01/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 08/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 15/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 22/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:955 [5] NCCL INFO Channel 29/0 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 02/0 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 03/0 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 09/0 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 10/0 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 16/0 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 17/0 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 23/0 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 24/0 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 30/0 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 31/0 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 02/0 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 02/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 03/0 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 09/0 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 04/0 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 10/0 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 05/0 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 16/0 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 17/0 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 11/0 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 23/0 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 24/0 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 30/0 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 12/0 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 31/0 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 18/0 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 19/0 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 25/0 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 26/0 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 04/0 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 09/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 05/0 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 11/0 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 12/0 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 18/0 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 19/0 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 25/0 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA comm 0x28cbfc0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 09/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 26/0 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA comm 0x28cbfc0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 02/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 16/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 06/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 09/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 23/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 12/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 13/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 30/0 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 04/0 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:958 [4] NCCL INFO Channel 16/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 11/0 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 16/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 20/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 18/0 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 19/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 23/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 25/0 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 23/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 27/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 26/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 30/0 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 30/0 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 03/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 04/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 05/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 06/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 11/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 10/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 12/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 13/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 18/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 17/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:958 [4] NCCL INFO Channel 03/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 19/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 04/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 25/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:958 [4] NCCL INFO Channel 10/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 03/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 02/0 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 26/0 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 20/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:958 [4] NCCL INFO Channel 11/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 04/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 09/0 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 06/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 24/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 16/0 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 10/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 13/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 27/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 23/0 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 02/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 11/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 20/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 31/0 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 30/0 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 09/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 27/0 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Channel 06/0 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 17/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 16/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 18/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 13/0 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:958 [4] NCCL INFO Channel 17/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 06/0 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 node1:918:962 [3] NCCL INFO Channel 23/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 13/0 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 20/0 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 24/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 27/0 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 node1:917:960 [2] NCCL INFO Channel 20/0 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 06/0 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 18/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 13/0 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 node1:918:962 [3] NCCL INFO Channel 30/0 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 20/0 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 27/0 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA comm 0x21dc490 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 25/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 24/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:960 [2] NCCL INFO Channel 27/0 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:962 [3] NCCL INFO Channel 05/0 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 31/0 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 05/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 25/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 12/0 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 06/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 31/0 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 19/0 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 12/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 26/0 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 02/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 13/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 05/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:960 [2] NCCL INFO Channel 05/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:961 [0] NCCL INFO Channel 19/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 04/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 00/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 01/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 07/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 09/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 08/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 14/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 15/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 12/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 21/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 20/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 22/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 28/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 11/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 29/0 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 00/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 12/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 01/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 07/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 08/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 26/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 19/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 14/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 15/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 21/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 22/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 28/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA comm 0x33b1de0 nRanks 16 master:5748:5789 [0] NCCL INFO Channel 29/0 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA comm 0x33b1de0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 18/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:916:956 [1] NCCL INFO Channel 16/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 27/0 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 26/0 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 25/0 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:962 [3] NCCL INFO Channel 02/0 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 03/0 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 19/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:962 [3] NCCL INFO Channel 09/0 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 10/0 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 16/0 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 17/0 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 23/0 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 24/0 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 23/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:962 [3] NCCL INFO Channel 30/0 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 31/0 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 02/0 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 03/0 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 09/0 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 10/0 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 16/0 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 17/0 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 23/0 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 26/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:962 [3] NCCL INFO Channel 24/0 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 30/0 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA comm 0x3a34fa0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 31/0 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 30/0 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:961 [0] NCCL INFO Channel 02/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 04/0 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 05/0 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 11/0 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 12/0 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 18/0 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 19/0 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 25/0 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 26/0 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:915:961 [0] NCCL INFO Channel 09/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 04/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:958 [4] NCCL INFO Channel 04/0 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 05/0 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 11/0 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 12/0 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 18/0 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 19/0 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 25/0 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA comm 0x2b30900 nRanks 16 node1:919:958 [4] NCCL INFO Channel 26/0 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA comm 0x2b30900 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 05/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 16/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 05/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:958 [4] NCCL INFO Channel 05/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:956 [1] NCCL INFO Channel 11/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:958 [4] NCCL INFO Channel 06/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 06/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 23/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 12/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 12/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:958 [4] NCCL INFO Channel 12/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:961 [0] NCCL INFO Channel 30/0 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 13/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 13/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:956 [1] NCCL INFO Channel 18/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 19/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 19/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:956 [1] NCCL INFO Channel 19/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:955 [5] NCCL INFO Channel 02/0 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 20/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 09/0 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 26/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 16/0 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 27/0 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 03/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 23/0 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:958 [4] NCCL INFO Channel 20/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 30/0 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:958 [4] NCCL INFO Channel 26/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:961 [0] NCCL INFO Channel 04/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 27/0 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:961 [0] NCCL INFO Channel 10/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 11/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 17/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 18/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 24/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 25/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 31/0 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 25/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 26/0 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 06/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 13/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 20/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:956 [1] NCCL INFO Channel 27/0 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:922:957 [7] NCCL INFO Channel 06/0 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 13/0 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 20/0 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 27/0 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 06/0 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 13/0 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 20/0 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 27/0 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 01/0 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 01/0 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 08/0 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 08/0 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 15/0 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 15/0 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 22/0 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 22/0 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 29/0 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 29/0 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 04/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:959 [6] NCCL INFO Channel 11/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:959 [6] NCCL INFO Channel 18/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:959 [6] NCCL INFO Channel 25/0 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:961 [0] NCCL INFO Channel 00/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 01/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 07/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 08/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 14/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 15/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 21/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 22/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 28/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 29/0 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 00/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 01/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 07/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 08/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 14/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 15/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 21/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 22/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 28/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA comm 0x37c66a0 nRanks 16 node1:915:961 [0] NCCL INFO Channel 29/0 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA comm 0x37c66a0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 02/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 06/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 02/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 09/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 06/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 13/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 09/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 16/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 04/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 13/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 20/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 05/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 16/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 23/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 11/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 04/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 20/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 12/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 27/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 05/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 23/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 18/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 30/0 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 19/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 27/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 11/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 03/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 02/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 25/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 30/0 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 12/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 05/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 05/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 26/0 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 03/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 18/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 10/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 02/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 09/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 05/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 12/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 19/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 10/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 05/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 25/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 17/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 12/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 09/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 19/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 26/0 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 17/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:957 [7] NCCL INFO Channel 24/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 19/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 12/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 26/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 24/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 16/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 05/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 31/0 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 26/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 19/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 06/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 05/0 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 31/0 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:955 [5] NCCL INFO Channel 12/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 23/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 12/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 12/0 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 13/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:955 [5] NCCL INFO Channel 16/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 05/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 19/0 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 19/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:955 [5] NCCL INFO Channel 19/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 06/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 26/0 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 20/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 26/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 26/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 03/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 12/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:959 [6] NCCL INFO Channel 27/0 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 13/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 04/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 30/0 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 19/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:957 [7] NCCL INFO Channel 10/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 11/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 20/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:955 [5] NCCL INFO Channel 23/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:957 [7] NCCL INFO Channel 17/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 26/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:959 [6] NCCL INFO Channel 02/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:958 [4] NCCL INFO Channel 03/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:957 [7] NCCL INFO Channel 18/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 27/0 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:959 [6] NCCL INFO Channel 09/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 24/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 26/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:958 [4] NCCL INFO Channel 10/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 05/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 16/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 03/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 05/0 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:962 [3] NCCL INFO Channel 06/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 17/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 30/0 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:957 [7] NCCL INFO Channel 25/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 23/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 06/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 24/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 12/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 31/0 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 12/0 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 30/0 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:955 [5] NCCL INFO Channel 03/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 13/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 10/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 31/0 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 06/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 19/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 04/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 10/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 20/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 11/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 13/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 26/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 18/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 17/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 27/0 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 20/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:957 [7] NCCL INFO Channel 25/0 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 19/0 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 02/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:955 [5] NCCL INFO Channel 24/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 06/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:955 [5] NCCL INFO Channel 27/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 09/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:955 [5] NCCL INFO Channel 31/0 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 13/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 26/0 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 16/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 03/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 20/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:959 [6] NCCL INFO Channel 23/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 03/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 04/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 10/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 13/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 10/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:959 [6] NCCL INFO Channel 27/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 17/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 11/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:955 [5] NCCL INFO Channel 03/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 30/0 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 24/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 17/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 17/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:955 [5] NCCL INFO Channel 04/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 02/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 31/0 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 04/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 18/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 02/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 10/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 04/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 20/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 06/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 24/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 09/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 11/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 09/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:962 [3] NCCL INFO Channel 11/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 24/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 05/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 25/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:955 [5] NCCL INFO Channel 17/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:958 [4] NCCL INFO Channel 16/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:962 [3] NCCL INFO Channel 13/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 11/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 27/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 31/0 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:958 [4] NCCL INFO Channel 23/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 18/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 18/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 16/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 31/0 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 24/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:958 [4] NCCL INFO Channel 30/0 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 04/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:962 [3] NCCL INFO Channel 20/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 18/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:955 [5] NCCL INFO Channel 25/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 11/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:962 [3] NCCL INFO Channel 25/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 23/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:955 [5] NCCL INFO Channel 31/0 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 18/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:962 [3] NCCL INFO Channel 27/0 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 25/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:957 [7] NCCL INFO Channel 00/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 25/0 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:960 [2] NCCL INFO Channel 30/0 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:957 [7] NCCL INFO Channel 01/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 07/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 02/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 06/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 08/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 14/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 09/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 12/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 15/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 16/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 13/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 21/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 23/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 19/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 22/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 00/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 30/0 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 20/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:957 [7] NCCL INFO Channel 28/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 01/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:957 [7] NCCL INFO Channel 29/0 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 07/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:955 [5] NCCL INFO Channel 00/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:962 [3] NCCL INFO Channel 00/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:959 [6] NCCL INFO Channel 08/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 02/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 00/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:955 [5] NCCL INFO Channel 01/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 14/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 26/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 09/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 03/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:962 [3] NCCL INFO Channel 01/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 07/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 15/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 16/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 27/0 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 07/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 08/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:956 [1] NCCL INFO Channel 07/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:959 [6] NCCL INFO Channel 21/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 23/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 10/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:962 [3] NCCL INFO Channel 08/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:955 [5] NCCL INFO Channel 14/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 00/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:921:959 [6] NCCL INFO Channel 22/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 02/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 30/0 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 14/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 14/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:955 [5] NCCL INFO Channel 15/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 01/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:921:959 [6] NCCL INFO Channel 28/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 06/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:962 [3] NCCL INFO Channel 15/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 17/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:955 [5] NCCL INFO Channel 21/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:959 [6] NCCL INFO Channel 29/0 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:960 [2] NCCL INFO Channel 07/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 03/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:962 [3] NCCL INFO Channel 21/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 09/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 02/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 21/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:955 [5] NCCL INFO Channel 22/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 08/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:962 [3] NCCL INFO Channel 22/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 04/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 04/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 13/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:958 [4] NCCL INFO Channel 00/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:955 [5] NCCL INFO Channel 28/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:956 [1] NCCL INFO Channel 24/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:960 [2] NCCL INFO Channel 14/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:962 [3] NCCL INFO Channel 28/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 10/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 09/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 01/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:956 [1] NCCL INFO Channel 28/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:955 [5] NCCL INFO Channel 29/0 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:960 [2] NCCL INFO Channel 15/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 16/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:962 [3] NCCL INFO Channel 29/0 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 11/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 07/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 11/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:956 [1] NCCL INFO Channel 31/0 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:960 [2] NCCL INFO Channel 21/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 20/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 17/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 08/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 16/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 22/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 23/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:958 [4] NCCL INFO Channel 14/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 18/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 18/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 28/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:958 [4] NCCL INFO Channel 15/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 27/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 24/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 23/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:960 [2] NCCL INFO Channel 29/0 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:958 [4] NCCL INFO Channel 21/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 25/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 25/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 30/0 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:958 [4] NCCL INFO Channel 22/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 30/0 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 04/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 31/0 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 28/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 06/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:958 [4] NCCL INFO Channel 29/0 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 00/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 11/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 01/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 13/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 07/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 18/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 08/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 20/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 14/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 25/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 15/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 21/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 27/0 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 22/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 28/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5755:5793 [7] NCCL INFO Channel 29/0 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 00/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 00/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 00/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 01/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 01/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 07/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 01/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 07/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 08/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 07/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 08/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 14/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 08/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 14/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 15/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 15/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 14/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 21/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 21/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 15/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 22/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 22/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 28/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 21/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 28/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5792 [2] NCCL INFO Channel 29/0 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 22/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 00/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 00/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5796 [4] NCCL INFO Channel 29/0 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 28/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 01/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 01/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5794 [6] NCCL INFO Channel 29/0 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 07/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 07/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:959 [6] NCCL INFO Connected all trees node1:921:959 [6] NCCL INFO NCCL_ALGO set by environment to tree,ring node1:921:959 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 node1:921:959 [6] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5749:5790 [1] NCCL INFO Channel 00/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 08/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 08/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 03/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 14/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 07/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 14/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 15/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 10/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 15/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 21/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 14/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 21/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 22/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 17/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 22/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 28/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 21/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:960 [2] NCCL INFO Connected all trees node1:917:960 [2] NCCL INFO NCCL_ALGO set by environment to tree,ring node1:917:960 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 node1:917:960 [2] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer node1:916:956 [1] NCCL INFO Connected all trees node1:916:956 [1] NCCL INFO NCCL_ALGO set by environment to tree,ring node1:916:956 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 node1:916:956 [1] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5751:5795 [3] NCCL INFO Channel 28/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5791 [5] NCCL INFO Channel 29/0 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 24/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5795 [3] NCCL INFO Channel 29/0 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5790 [1] NCCL INFO Channel 28/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:955 [5] NCCL INFO Connected all trees node1:920:955 [5] NCCL INFO NCCL_ALGO set by environment to tree,ring node1:920:955 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 node1:920:955 [5] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5749:5790 [1] NCCL INFO Channel 31/0 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5794 [6] NCCL INFO Connected all trees master:5754:5794 [6] NCCL INFO NCCL_ALGO set by environment to tree,ring master:5754:5794 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 master:5754:5794 [6] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5753:5791 [5] NCCL INFO Connected all trees master:5753:5791 [5] NCCL INFO NCCL_ALGO set by environment to tree,ring master:5753:5791 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 master:5753:5791 [5] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5750:5792 [2] NCCL INFO Connected all trees master:5750:5792 [2] NCCL INFO NCCL_ALGO set by environment to tree,ring master:5750:5792 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 master:5750:5792 [2] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5749:5790 [1] NCCL INFO Connected all trees master:5749:5790 [1] NCCL INFO NCCL_ALGO set by environment to tree,ring master:5749:5790 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 master:5749:5790 [1] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer node1:922:957 [7] NCCL INFO Connected all trees node1:922:957 [7] NCCL INFO NCCL_ALGO set by environment to tree,ring node1:922:957 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 node1:922:957 [7] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5755:5793 [7] NCCL INFO Connected all trees master:5755:5793 [7] NCCL INFO NCCL_ALGO set by environment to tree,ring master:5755:5793 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 master:5755:5793 [7] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer node1:919:958 [4] NCCL INFO Connected all trees node1:919:958 [4] NCCL INFO NCCL_ALGO set by environment to tree,ring node1:919:958 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 node1:919:958 [4] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5752:5796 [4] NCCL INFO Connected all trees master:5752:5796 [4] NCCL INFO NCCL_ALGO set by environment to tree,ring master:5752:5796 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 master:5752:5796 [4] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer node1:918:962 [3] NCCL INFO Connected all trees node1:918:962 [3] NCCL INFO NCCL_ALGO set by environment to tree,ring node1:918:962 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 node1:918:962 [3] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer node1:915:961 [0] NCCL INFO Connected all trees node1:915:961 [0] NCCL INFO NCCL_ALGO set by environment to tree,ring node1:915:961 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 node1:915:961 [0] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5751:5795 [3] NCCL INFO Connected all trees master:5751:5795 [3] NCCL INFO NCCL_ALGO set by environment to tree,ring master:5751:5795 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 master:5751:5795 [3] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer master:5748:5789 [0] NCCL INFO Connected all trees master:5748:5789 [0] NCCL INFO NCCL_ALGO set by environment to tree,ring master:5748:5789 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 master:5748:5789 [0] NCCL INFO 32 coll channels, 0 nvls channels, 32 p2p channels, 32 p2p channels per peer node1:919:958 [4] NCCL INFO Init config for nccl_context_test: 0 node1:915:961 [0] NCCL INFO Init config for nccl_context_test: 0 node1:918:962 [3] NCCL INFO Init config for nccl_context_test: 0 node1:921:959 [6] NCCL INFO Init config for nccl_context_test: 0 node1:922:957 [7] NCCL INFO Init config for nccl_context_test: 0 node1:920:955 [5] NCCL INFO Init config for nccl_context_test: 0 node1:916:956 [1] NCCL INFO Init config for nccl_context_test: 0 node1:917:960 [2] NCCL INFO Init config for nccl_context_test: 0 node1:919:958 [4] NCCL INFO Maximum number of GPUs in any NUMA node: 2 node1:919:958 [4] NCCL INFO comm 0x2b30900 rank 12 nranks 16 cudaDev 4 busId e8000 commId 0x2c23323128873473 localSize 464 used 1234059600 bytes - Init COMPLETE node1:915:961 [0] NCCL INFO Maximum number of GPUs in any NUMA node: 2 node1:915:961 [0] NCCL INFO comm 0x37c66a0 rank 8 nranks 16 cudaDev 0 busId 9f000 commId 0x2c23323128873473 localSize 464 used 1242054992 bytes - Init COMPLETE node1:918:962 [3] NCCL INFO Maximum number of GPUs in any NUMA node: 2 node1:918:962 [3] NCCL INFO comm 0x3a34fa0 rank 11 nranks 16 cudaDev 3 busId 5000 commId 0x2c23323128873473 localSize 464 used 1252540752 bytes - Init COMPLETE node1:921:959 [6] NCCL INFO Maximum number of GPUs in any NUMA node: 2 node1:922:957 [7] NCCL INFO Maximum number of GPUs in any NUMA node: 2 node1:921:959 [6] NCCL INFO comm 0x26c3520 rank 14 nranks 16 cudaDev 6 busId ca000 commId 0x2c23323128873473 localSize 464 used 1086734672 bytes - Init COMPLETE node1:920:955 [5] NCCL INFO Maximum number of GPUs in any NUMA node: 2 node1:920:955 [5] NCCL INFO comm 0x2e85470 rank 13 nranks 16 cudaDev 5 busId c1000 commId 0x2c23323128873473 localSize 464 used 1095123280 bytes - Init COMPLETE node1:916:956 [1] NCCL INFO Maximum number of GPUs in any NUMA node: 2 node1:917:960 [2] NCCL INFO Maximum number of GPUs in any NUMA node: 2 node1:917:960 [2] NCCL INFO comm 0x23c8a20 rank 10 nranks 16 cudaDev 2 busId 5d000 commId 0x2c23323128873473 localSize 464 used 1143357776 bytes - Init COMPLETE node1:922:957 [7] NCCL INFO comm 0x38568d0 rank 15 nranks 16 cudaDev 7 busId b1000 commId 0x2c23323128873473 localSize 464 used 937050448 bytes - Init COMPLETE node1:916:956 [1] NCCL INFO comm 0x264c650 rank 9 nranks 16 cudaDev 1 busId 56000 commId 0x2c23323128873473 localSize 464 used 1128677712 bytes - Init COMPLETE master:5751:5795 [3] NCCL INFO Init config for nccl_context_test: 0 master:5750:5792 [2] NCCL INFO Init config for nccl_context_test: 0 master:5755:5793 [7] NCCL INFO Init config for nccl_context_test: 0 master:5753:5791 [5] NCCL INFO Init config for nccl_context_test: 0 master:5752:5796 [4] NCCL INFO Init config for nccl_context_test: 0 master:5748:5789 [0] NCCL INFO Init config for nccl_context_test: 0 master:5754:5794 [6] NCCL INFO Init config for nccl_context_test: 0 master:5749:5790 [1] NCCL INFO Init config for nccl_context_test: 0 master:5751:5795 [3] NCCL INFO Maximum number of GPUs in any NUMA node: 2 master:5751:5795 [3] NCCL INFO comm 0x24c05f0 rank 3 nranks 16 cudaDev 3 busId 5000 commId 0x2c23323128873473 localSize 464 used 1252540752 bytes - Init COMPLETE master:5750:5792 [2] NCCL INFO Maximum number of GPUs in any NUMA node: 2 master:5750:5792 [2] NCCL INFO comm 0x3d308b0 rank 2 nranks 16 cudaDev 2 busId 5d000 commId 0x2c23323128873473 localSize 464 used 1143357776 bytes - Init COMPLETE master:5749:5790 [1] NCCL INFO Maximum number of GPUs in any NUMA node: 2 master:5749:5790 [1] NCCL INFO comm 0x257e160 rank 1 nranks 16 cudaDev 1 busId 56000 commId 0x2c23323128873473 localSize 464 used 1128677712 bytes - Init COMPLETE master:5748:5789 [0] NCCL INFO Maximum number of GPUs in any NUMA node: 2 master:5748:5789 [0] NCCL INFO comm 0x33b1de0 rank 0 nranks 16 cudaDev 0 busId 9f000 commId 0x2c23323128873473 localSize 464 used 1242054992 bytes - Init COMPLETE master:5754:5794 [6] NCCL INFO Maximum number of GPUs in any NUMA node: 2 master:5754:5794 [6] NCCL INFO comm 0x2fbfb50 rank 6 nranks 16 cudaDev 6 busId ca000 commId 0x2c23323128873473 localSize 464 used 1086734672 bytes - Init COMPLETE master:5752:5796 [4] NCCL INFO Maximum number of GPUs in any NUMA node: 2 master:5755:5793 [7] NCCL INFO Maximum number of GPUs in any NUMA node: 2 master:5753:5791 [5] NCCL INFO Maximum number of GPUs in any NUMA node: 2 master:5752:5796 [4] NCCL INFO comm 0x28cbfc0 rank 4 nranks 16 cudaDev 4 busId e8000 commId 0x2c23323128873473 localSize 464 used 1234059600 bytes - Init COMPLETE master:5753:5791 [5] NCCL INFO comm 0x3e928f0 rank 5 nranks 16 cudaDev 5 busId c1000 commId 0x2c23323128873473 localSize 464 used 1095123280 bytes - Init COMPLETE master:5755:5793 [7] NCCL INFO comm 0x21dc490 rank 7 nranks 16 cudaDev 7 busId b1000 commId 0x2c23323128873473 localSize 464 used 937050448 bytes - Init COMPLETE # # out-of-place in-place # size count type redop root time algbw busbw #wrong time algbw busbw #wrong # (B) (elements) (us) (GB/s) (GB/s) (us) (GB/s) (GB/s) master:5748:5877 [0] NCCL INFO Channel 00/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 15[b1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 0[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 7[b1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 8[9f000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 15[b1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 12[e8000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 14[ca000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 3[5000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 1[56000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 10[5d000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 4[e8000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 5[c1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 6[ca000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 2[5d000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 13[c1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 11[5000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 14[ca000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 9[56000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 7[b1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 6[ca000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 0[9f000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 15[b1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 1[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 1[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 7[b1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 10[5d000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 2[5d000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 8[9f000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 9[56000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 9[56000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 4[e8000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 14[ca000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 13[c1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 5[c1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 13[c1000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 15[b1000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 6[ca000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 3[5000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 12[e8000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 15[b1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 7[b1000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 11[5000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 14[ca000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 7[b1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 5[c1000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 6[ca000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 1[56000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 1[56000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 5[c1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 3[5000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 0[9f000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 13[c1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 4[e8000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 2[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 12[e8000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 11[5000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 9[56000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 9[56000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 8[9f000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 10[5d000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 15[b1000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 7[b1000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 14[ca000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 5[c1000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 13[c1000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 4[e8000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 6[ca000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 12[e8000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 14[ca000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 13[c1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 0[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 12[e8000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 2[5d000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 5[c1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 8[9f000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 10[5d000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 6[ca000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 3[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 3[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 13[c1000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 4[e8000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 9[56000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 9[56000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 15[b1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 7[b1000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 7[b1000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 6[ca000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 10[5d000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 14[ca000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 11[5000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 11[5000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 5[c1000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 11[5000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 14[ca000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 8[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 8[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 15[b1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 9[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 9[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 7[b1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 3[5000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 7[b1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 1[56000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 1[56000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 1[56000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 1[56000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 15[b1000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 12[e8000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 4[e8000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 5[c1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 15[b1000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 13[c1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 6[ca000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 0[9f000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 0[9f000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 3[5000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 3[5000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 11[5000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 11[5000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 13[c1000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 10[5d000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 5[c1000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 9[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 9[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 1[56000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 1[56000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 14[ca000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 2[5d000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 2[5d000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 6[ca000] via P2P/IPC comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 12[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 4[e8000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 12[e8000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 6[ca000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 4[e8000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 14[ca000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 2[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 8[9f000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 0[9f000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 10[5d000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 6[ca000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 3[5000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 1[56000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 1[56000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 4[e8000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 14[ca000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 1[56000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 3[5000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 12[e8000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 3[5000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 11[5000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 13[c1000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 7[b1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 9[56000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 9[56000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 9[56000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 13[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 11[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 11[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 5[c1000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 3[5000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 3[5000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 3[5000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 3[5000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 6[ca000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 5[c1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 4[e8000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 15[b1000] via P2P/IPC comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 7[b1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 5[c1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 15[b1000] via P2P/IPC comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 2[5d000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 12[e8000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 7[b1000] via P2P/IPC comm 0x33b1de0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 0[9f000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 0[9f000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 10[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 10[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 11[5000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 11[5000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 15[b1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 11[5000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 11[5000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 14[ca000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 12[e8000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 13[c1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 7[b1000] via P2P/IPC comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 4[e8000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 10[5d000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 11[5000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 11[5000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 2[5d000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 2[5d000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 8[9f000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 11[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 8[9f000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 11[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 4[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 6[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 9[56000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 8[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 9[56000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 12[e8000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 2[5d000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 0[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 9[56000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 9[56000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 15[b1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 13[c1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 1[56000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 1[56000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 10[5d000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 1[56000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 1[56000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 14[ca000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 9[56000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 0[9f000] -> 9[56000] [receive] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 1[56000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 8[9f000] -> 1[56000] [receive] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 11[5000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 11[5000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 2[5d000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 3[5000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 2[5d000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 3[5000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 10[5d000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 10[5d000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 11[5000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 5[c1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 13[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 14[ca000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 12[e8000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 4[e8000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 3[5000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 0[9f000] [receive] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 3[5000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 14[ca000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 6[ca000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 3[5000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 8[9f000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 3[5000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 2[5d000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 10[5d000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 3[5000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 4[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 4[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 5[c1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 1[56000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 1[56000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 12[e8000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 12[e8000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 8[9f000] -> 2[5d000] [receive] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 6[ca000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 10[5d000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 5[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 5[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 7[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 12[e8000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 9[56000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 0[9f000] -> 10[5d000] [receive] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 9[56000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 2[5d000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 14[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 5[c1000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 13[c1000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 4[e8000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 13[c1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 13[c1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 6[ca000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 7[b1000] [send] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 15[b1000] [send] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 1[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 1[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 9[56000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 9[56000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 15[b1000] [send] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 14[ca000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 6[ca000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 3[5000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 3[5000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 11[5000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 11[5000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 0[9f000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 8[9f000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 13[c1000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 4[e8000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 11[5000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 5[c1000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 15[b1000] [send] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 0[9f000] -> 11[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 12[e8000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 6[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 6[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 3[5000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 7[b1000] [send] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 8[9f000] -> 3[5000] [receive] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 1[56000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 14[ca000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 14[ca000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 2[5d000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 12[e8000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 9[56000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 8[9f000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 2[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 10[5d000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 9[56000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 4[e8000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 0[9f000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 1[56000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 7[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 15[b1000] [send] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 10[5d000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 8[9f000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 2[5d000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 12[e8000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 12[e8000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 0[9f000] -> 12[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 3[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 3[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 10[5d000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 4[e8000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 13[c1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 4[e8000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 1[56000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 0[9f000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/4/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 8[9f000] -> 4[e8000] [receive] via NET/IB/5/GDRDMA/Shared comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 3[5000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 5[c1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 11[5000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 11[5000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 11[5000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 8[9f000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 9[56000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 4[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 9[56000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 14[ca000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 11[5000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 0[9f000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 9[56000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 2[5d000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 6[ca000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 1[56000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 10[5d000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 1[56000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 12[e8000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 3[5000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 10[5d000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 5[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 1[56000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 5[c1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 5[c1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 15[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 15[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 7[b1000] [send] via NET/IB/2/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 7[b1000] [send] via NET/IB/3/GDRDMA/Shared comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 2[5d000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 10[5d000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 9[56000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 13[c1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 13[c1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/2/Shared comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 0[9f000] -> 13[c1000] [receive] via NET/IB/6/Shared comm 0x2e85470 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 11[5000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/2/Shared comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 8[9f000] -> 5[c1000] [receive] via NET/IB/6/Shared comm 0x3e928f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 13[c1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 3[5000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 6[ca000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 6[ca000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 9[56000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 3[5000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/1/Shared comm 0x23c8a20 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 7[b1000] [send] via NET/IB/5/Shared comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 4[e8000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 1[56000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 10[5d000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/1/Shared comm 0x3d308b0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 15[b1000] [send] via NET/IB/5/Shared comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 9[56000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 2[5d000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 14[ca000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 14[ca000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 11[5000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 2[5d000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 7[b1000] [send] via NET/IB/0/Shared comm 0x264c650 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 7[b1000] [send] via NET/IB/4/Shared comm 0x264c650 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 5[c1000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 15[b1000] [send] via NET/IB/0/Shared comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 12[e8000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 15[b1000] [send] via NET/IB/4/Shared comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 1[56000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 11[5000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 8[9f000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 8[9f000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 3[5000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 2[5d000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 0[9f000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/3/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 0[9f000] -> 14[ca000] [receive] via NET/IB/7/Shared comm 0x26c3520 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 8[9f000] -> 7[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x21dc490 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 00/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 01/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/3/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 8[9f000] -> 6[ca000] [receive] via NET/IB/7/Shared comm 0x2fbfb50 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 02/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 03/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 13[c1000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 04/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 05/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 06/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 07/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 08/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/6/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 0[9f000] -> 15[b1000] [receive] via NET/IB/7/GDRDMA/Shared comm 0x38568d0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 00/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 09/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 01/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 10/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 02/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 03/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 11/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 04/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 12/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 05/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 13/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 06/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 14/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 07/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 15/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 08/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 16/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 09/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 10/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 17/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 11/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 18/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 12/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 19/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 13/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 20/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 14/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 21/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 15/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 22/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 16/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 23/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 17/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 24/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 18/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 19/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 20/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 0[9f000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 21/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 22/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 23/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 25/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 10[5d000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 24/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 26/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 25/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 26/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 27/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 28/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 27/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 29/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 28/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 29/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 30/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 30/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 master:5755:5883 [7] NCCL INFO Channel 31/1 : 7[b1000] -> 6[ca000] via P2P/IPC comm 0x21dc490 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:922:1050 [7] NCCL INFO Channel 31/1 : 15[b1000] -> 14[ca000] via P2P/IPC comm 0x38568d0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 14[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 12[e8000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 4[e8000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 1[56000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 6[ca000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 9[56000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 00/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 00/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 01/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 01/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 02/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 02/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 03/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 03/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 04/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 04/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 05/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 05/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 06/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 00/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 06/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 01/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 02/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 03/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 07/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 04/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 05/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 06/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 00/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 07/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 07/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 08/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 09/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 10/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 11/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 12/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 08/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 13/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 14/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 15/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 16/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 01/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 17/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 08/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 18/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 19/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 20/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 21/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 22/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 09/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 23/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 24/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 25/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 26/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 02/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 09/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 27/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 28/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 29/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 30/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x33b1de0 nRanks 16 master:5748:5877 [0] NCCL INFO Channel 31/1 : 0[9f000] -> 15[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x33b1de0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 10/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 03/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 10/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 11/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 04/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 11/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 12/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 05/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 12/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 13/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 06/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 14/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 13/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 07/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 15/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 14/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 08/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 00/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 16/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 01/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 00/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 02/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 15/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 03/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 04/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 05/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 06/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 07/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 08/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 17/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 09/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 09/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 10/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 11/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 01/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 12/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 16/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 13/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 14/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 15/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 16/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 18/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 17/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 10/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 18/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 19/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 00/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 20/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 02/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 21/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 17/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 22/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 23/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 24/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 00/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 25/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 19/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 26/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 11/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 27/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 28/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 29/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 01/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 30/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/0/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 03/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 18/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:915:1045 [0] NCCL INFO Channel 31/1 : 8[9f000] -> 7[b1000] [send] via NET/IB/1/GDRDMA/Shared comm 0x37c66a0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 01/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 20/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 00/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 12/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 02/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 04/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 19/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 21/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 02/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 01/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 13/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 20/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 03/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 05/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 22/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 02/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 03/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 14/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 21/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 23/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 06/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 00/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 04/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 03/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 04/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 15/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 24/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 22/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 07/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 04/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 01/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 16/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 05/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 00/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 05/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 25/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 23/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 05/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 02/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 08/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 17/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 01/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 06/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 06/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 26/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 24/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 06/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 03/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 09/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 18/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 02/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 07/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 07/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 27/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 25/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 00/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 07/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 04/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 19/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 10/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 03/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 28/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 08/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 08/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 26/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 08/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 01/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 05/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 20/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 29/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 11/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 04/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 09/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 09/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 27/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 09/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 02/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 06/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 21/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 30/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 12/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 05/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 10/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 10/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 28/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 10/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 00/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 22/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 03/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:921:1044 [6] NCCL INFO Channel 31/1 : 14[ca000] -> 13[c1000] via P2P/IPC comm 0x26c3520 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 07/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 13/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 06/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 11/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 29/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 11/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 01/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 11/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 23/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 08/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 04/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 12/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 07/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 14/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 30/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 12/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 02/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 24/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 09/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 12/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 05/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 13/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 08/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 15/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 00/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5750:5878 [2] NCCL INFO Channel 31/1 : 2[5d000] -> 1[56000] via P2P/IPC comm 0x3d308b0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 13/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 03/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 25/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 10/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 13/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 06/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 14/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 01/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 09/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 16/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 14/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 14/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 07/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 10/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 17/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 15/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 15/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 08/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 11/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 18/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 16/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 16/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 09/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 12/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 19/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 17/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 17/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 10/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 13/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 20/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 18/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 18/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 11/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 14/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 21/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 19/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 19/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 12/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 15/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 22/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 20/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 20/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 13/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 16/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 23/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 26/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 04/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 15/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 02/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 21/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 11/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 21/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 14/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 17/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 27/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 24/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 16/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 05/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 03/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 12/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 22/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 22/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 15/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 18/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 28/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 17/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 25/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 06/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 04/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 13/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 23/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 23/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 19/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 16/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 29/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 18/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 26/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 07/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 05/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 14/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 24/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 24/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 20/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 30/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 17/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 19/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 27/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 08/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 06/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 15/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 25/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:919:1043 [4] NCCL INFO Channel 31/1 : 12[e8000] -> 11[5000] via P2P/IPC comm 0x2b30900 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 25/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 20/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 21/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 18/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 07/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 28/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 16/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 09/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 26/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 21/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 26/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 22/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 08/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 19/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 29/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 27/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 10/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 17/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 22/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 09/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 23/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 27/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 20/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 30/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 18/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 28/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 11/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 23/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 10/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 24/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 28/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 21/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5752:5880 [4] NCCL INFO Channel 31/1 : 4[e8000] -> 3[5000] via P2P/IPC comm 0x28cbfc0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 12/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 19/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 29/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 24/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 11/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 25/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 29/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 22/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 30/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 20/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 13/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 25/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 12/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 26/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 30/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 23/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 master:5751:5884 [3] NCCL INFO Channel 31/1 : 3[5000] -> 2[5d000] via P2P/IPC comm 0x24c05f0 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 14/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 26/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 21/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 13/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 27/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5754:5879 [6] NCCL INFO Channel 31/1 : 6[ca000] -> 5[c1000] via P2P/IPC comm 0x2fbfb50 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 24/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 22/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 15/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 27/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 14/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 28/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 25/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 28/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 23/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 16/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 15/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 29/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 29/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 26/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 24/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 17/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 16/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 30/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 30/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 25/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 27/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 18/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 17/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:918:1049 [3] NCCL INFO Channel 31/1 : 11[5000] -> 10[5d000] via P2P/IPC comm 0x3a34fa0 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 28/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 18/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 26/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 19/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 29/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 19/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 27/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 20/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 30/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 28/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 20/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 21/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5749:5882 [1] NCCL INFO Channel 31/1 : 1[56000] -> 0[9f000] via P2P/IPC comm 0x257e160 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 21/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 29/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 22/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 30/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 22/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 23/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 23/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 24/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:917:1046 [2] NCCL INFO Channel 31/1 : 10[5d000] -> 9[56000] via P2P/IPC comm 0x23c8a20 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 24/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 25/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 25/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 26/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 26/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 27/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 master:5753:5881 [5] NCCL INFO Channel 31/1 : 5[c1000] -> 4[e8000] via P2P/IPC comm 0x3e928f0 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 27/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 28/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 28/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 29/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 29/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 30/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 30/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 node1:916:1048 [1] NCCL INFO Channel 31/1 : 9[56000] -> 8[9f000] via P2P/IPC comm 0x264c650 nRanks 16 node1:920:1047 [5] NCCL INFO Channel 31/1 : 13[c1000] -> 12[e8000] via P2P/IPC comm 0x2e85470 nRanks 16 master:5749:5835 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error master:5749:5835 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:917:1015 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1009 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:916:995 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:916:995 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:916:995 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:917:1015 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error master:5749:5835 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1009 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error master:5752:5839 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:918:984 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:918:984 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error master:5752:5839 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:918:987 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error master:5749:5835 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:918:984 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:920:973 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:994 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:918:987 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:917:1015 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:917:1015 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:917:1015 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error node1:919:1013 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:102 NCCL WARN NET/IB : Got async event : local access violation work queue error master:5755:5870 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.196<40762> with error 12, opcode 0, len 0, vendor err 129 (Recv) master:5755:5870 [7] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 master:5755:5870 [7] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 master:5755:5870 [7] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] master:5751:5872 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.196<48080> with error 12, opcode 0, len 0, vendor err 129 (Recv) master:5751:5872 [3] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 master:5751:5872 [3] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 master:5751:5872 [3] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] master:5754:5871 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.196<53294> with error 12, opcode 0, len 0, vendor err 129 (Recv) master:5754:5871 [6] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 master:5754:5871 [6] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 master:5754:5871 [6] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] master:5753:5874 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.196<47482> with error 12, opcode 0, len 0, vendor err 129 (Recv) master:5753:5874 [5] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 master:5753:5874 [5] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 master:5753:5874 [5] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] master:5750:5876 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.196<45720> with error 12, opcode 0, len 0, vendor err 129 (Recv) master:5750:5876 [2] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 master:5750:5876 [2] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 master:5750:5876 [2] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] master:5748:5869 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.196<35088> with error 12, opcode 0, len 0, vendor err 129 (Recv) master:5748:5869 [0] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 master:5748:5869 [0] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 master:5748:5869 [0] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] master:5749:5875 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.196<44762> with error 12, opcode 0, len 0, vendor err 129 (Recv) master:5749:5875 [1] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 master:5749:5875 [1] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 master:5749:5875 [1] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] node1:922:1036 [7] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.195<40166> with error 12, opcode 0, len 0, vendor err 129 (Recv) node1:922:1036 [7] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 node1:922:1036 [7] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:922:1036 [7] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] node1:921:1037 [6] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.195<58864> with error 12, opcode 0, len 0, vendor err 129 (Recv) node1:921:1037 [6] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 node1:921:1037 [6] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:921:1037 [6] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] node1:920:1040 [5] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.195<36250> with error 12, opcode 0, len 0, vendor err 129 (Recv) node1:920:1040 [5] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 node1:920:1040 [5] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:920:1040 [5] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] node1:915:1035 [0] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.195<39086> with error 12, opcode 0, len 0, vendor err 129 (Recv) node1:915:1035 [0] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 node1:915:1035 [0] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:915:1035 [0] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] node1:917:1042 [2] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.195<56714> with error 12, opcode 0, len 0, vendor err 129 (Recv) node1:917:1042 [2] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 node1:917:1042 [2] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:917:1042 [2] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] master:5752:5873 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.196<46114> with error 12, opcode 0, len 0, vendor err 129 (Recv) master:5752:5873 [4] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 master:5752:5873 [4] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:918:1038 [3] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.195<42306> with error 12, opcode 0, len 0, vendor err 129 (Recv) node1:918:1038 [3] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 node1:918:1038 [3] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:918:1038 [3] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] node1:916:1041 [1] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.195<57226> with error 12, opcode 0, len 0, vendor err 129 (Recv) node1:916:1041 [1] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 node1:916:1041 [1] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:916:1041 [1] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] node1:919:1039 [4] /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net_ib.cc:1332 NCCL WARN NET/IB : Got completion from peer 10.17.26.195<60472> with error 12, opcode 0, len 0, vendor err 129 (Recv) node1:919:1039 [4] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/transport/net.cc:1286 -> 6 node1:919:1039 [4] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:689 -> 6 node1:919:1039 [4] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread] master:5752:5873 [4] NCCL INFO /data/jenkins_workspace/workspace/rccl_release/build/hipify/src/proxy.cc:868 -> 6 [Proxy Thread]