{"node": "sb-validation-01","bert_models/pytorch-bert-base/steptime_train_float32": 114.59167010616511,"bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591105, "bert_models/pytorch-bert-base/steptime_train_float16": 83.88951083179563, "bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582741963, "bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371913597, "bert_models/pytorch-bert-large/throughput_train_float32": 103.94876097417632, "bert_models/pytorch-bert-large/steptime_train_float16": 206.81141689419746, "bert_models/pytorch-bert-large/throughput_train_float16": 154.84089117113942, "pytorch-computation-communication-overlap/mul_cost:0": 44.182206214372854, "pytorch-computation-communication-overlap/mul_cost:1": 44.18221393893873, "pytorch-computation-communication-overlap/mul_cost:2": 43.970147078084665, "pytorch-computation-communication-overlap/mul_cost:3": 43.97014787559783, "pytorch-computation-communication-overlap/mul_cost:4": 43.97017793166924, "pytorch-computation-communication-overlap/mul_cost:5": 43.97015716062924, "pytorch-computation-communication-overlap/mul_cost:6": 43.97016519828867, "pytorch-computation-communication-overlap/mul_cost:7": 44.179544478538446, "pytorch-computation-communication-overlap/matmul_cost:0": 137.04773705558182, "pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344692856, "pytorch-computation-communication-overlap/matmul_cost:2": 137.04777220902997, "pytorch-computation-communication-overlap/matmul_cost:3": 137.04779697263803, "pytorch-computation-communication-overlap/matmul_cost:4": 137.04813674314664, "pytorch-computation-communication-overlap/matmul_cost:5": 137.04821988767435, "pytorch-computation-communication-overlap/matmul_cost:6": 137.04775322366913, "pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081606734, "densenet_models/pytorch-densenet169/steptime_train_float32": 150.64155543223023, "densenet_models/pytorch-densenet169/throughput_train_float32": 212.47017192425312, "densenet_models/pytorch-densenet169/steptime_train_float16": 145.30819572973996, "densenet_models/pytorch-densenet169/throughput_train_float16": 220.293233730535, "densenet_models/pytorch-densenet201/steptime_train_float32": 182.91117786429822, "densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232251, "densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596353, "densenet_models/pytorch-densenet201/throughput_train_float16": 181.54465736033902, "gemm-flops/FP64:0": 9031.23, "gemm-flops/FP64:1": 9040.85, "gemm-flops/FP64:2": 9010.56, "gemm-flops/FP64:3": 9041.26, "gemm-flops/FP64:4": 9039.19, "gemm-flops/FP64:5": 9015.69, "gemm-flops/FP64:6": 9022.19, "gemm-flops/FP64:7": 9030.2, "gemm-flops/FP32:0": 18362.1, "gemm-flops/FP32:1": 18375.6, "gemm-flops/FP32:2": 18314.9, "gemm-flops/FP32:3": 18375.6, "gemm-flops/FP32:4": 18368.6, "gemm-flops/FP32:5": 18347.1, "gemm-flops/FP32:6": 18247.4, "gemm-flops/FP32:7": 18318.4, "gemm-flops/FP16:0": 33878.0, "gemm-flops/FP16:1": 33911.1, "gemm-flops/FP16:2": 33769.3, "gemm-flops/FP16:3": 33909.9, "gemm-flops/FP16:4": 33896.5, "gemm-flops/FP16:5": 33798.1, "gemm-flops/FP16:6": 33647.3, "gemm-flops/FP16:7": 33764.8, "gemm-flops/FP64_TC:0": 18963.6, "gemm-flops/FP64_TC:1": 18924.2, "gemm-flops/FP64_TC:2": 18930.3, "gemm-flops/FP64_TC:3": 18971.9, "gemm-flops/FP64_TC:4": 18946.0, "gemm-flops/FP64_TC:5": 18945.0, "gemm-flops/FP64_TC:6": 18822.9, "gemm-flops/FP64_TC:7": 18911.1, "gemm-flops/TF32_TC:0": 127900.0, "gemm-flops/TF32_TC:1": 129094.0, "gemm-flops/TF32_TC:2": 127831.0, "gemm-flops/TF32_TC:3": 128709.0, "gemm-flops/TF32_TC:4": 127388.0, "gemm-flops/TF32_TC:5": 127861.0, "gemm-flops/TF32_TC:6": 128492.0, "gemm-flops/TF32_TC:7": 127720.0, "gemm-flops/BF16_TC:0": 264965.0, "gemm-flops/BF16_TC:1": 266638.0, "gemm-flops/BF16_TC:2": 263151.0, "gemm-flops/BF16_TC:3": 264752.0, "gemm-flops/BF16_TC:4": 263049.0, "gemm-flops/BF16_TC:5": 266605.0, "gemm-flops/BF16_TC:6": 267501.0, "gemm-flops/BF16_TC:7": 263880.0, "gemm-flops/FP16_TC:0": 279474.0, "gemm-flops/FP16_TC:1": 281256.0, "gemm-flops/FP16_TC:2": 277403.0, "gemm-flops/FP16_TC:3": 279147.0, "gemm-flops/FP16_TC:4": 277587.0, "gemm-flops/FP16_TC:5": 281537.0, "gemm-flops/FP16_TC:6": 282132.0, "gemm-flops/FP16_TC:7": 277788.0, "gemm-flops/INT8_TC:0": 475160.0, "gemm-flops/INT8_TC:1": 477725.0, "gemm-flops/INT8_TC:2": 471621.0, "gemm-flops/INT8_TC:3": 473716.0, "gemm-flops/INT8_TC:4": 472124.0, "gemm-flops/INT8_TC:5": 479972.0, "gemm-flops/INT8_TC:6": 481327.0, "gemm-flops/INT8_TC:7": 474710.0, "gemm-flops/INT4_TC:0": 970330.0, "gemm-flops/INT4_TC:1": 976837.0, "gemm-flops/INT4_TC:2": 966003.0, "gemm-flops/INT4_TC:3": 971315.0, "gemm-flops/INT4_TC:4": 964441.0, "gemm-flops/INT4_TC:5": 982461.0, "gemm-flops/INT4_TC:6": 979610.0, "gemm-flops/INT4_TC:7": 968359.0, "gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971835643, "gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.115454396866486, "gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742234692, "gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.139449941061585, "gpu-sm-copy-bw/dtoh:0": 3.91755, "gpu-sm-copy-bw/dtoh:1": 4.45414, "gpu-sm-copy-bw/dtoh:2": 1.26483, "gpu-sm-copy-bw/dtoh:3": 1.30041, "gpu-sm-copy-bw/dtoh:4": 1.31577, "gpu-sm-copy-bw/dtoh:5": 1.27968, "gpu-sm-copy-bw/dtoh:6": 4.47849, "gpu-sm-copy-bw/dtoh:7": 3.96231, "gpu-sm-copy-bw/dtoh:8": 3.91705, "gpu-sm-copy-bw/dtoh:9": 4.45487, "gpu-sm-copy-bw/dtoh:10": 1.26352, "gpu-sm-copy-bw/dtoh:11": 1.2999, "gpu-sm-copy-bw/dtoh:12": 1.31677, "gpu-sm-copy-bw/dtoh:13": 1.27885, "gpu-sm-copy-bw/dtoh:14": 4.47913, "gpu-sm-copy-bw/dtoh:15": 3.95893, "gpu-sm-copy-bw/dtoh:16": 3.91729, "gpu-sm-copy-bw/dtoh:17": 4.45627, "gpu-sm-copy-bw/dtoh:18": 1.26437, "gpu-sm-copy-bw/dtoh:19": 1.30144, "gpu-sm-copy-bw/dtoh:20": 1.31704, "gpu-sm-copy-bw/dtoh:21": 1.27857, "gpu-sm-copy-bw/dtoh:22": 4.47889, "gpu-sm-copy-bw/dtoh:23": 3.95984, "gpu-sm-copy-bw/dtoh:24": 3.92025, "gpu-sm-copy-bw/dtoh:25": 4.45423, "gpu-sm-copy-bw/dtoh:26": 1.26449, "gpu-sm-copy-bw/dtoh:27": 1.29954, "gpu-sm-copy-bw/dtoh:28": 1.31731, "gpu-sm-copy-bw/dtoh:29": 1.27916, "gpu-sm-copy-bw/dtoh:30": 4.4797, "gpu-sm-copy-bw/dtoh:31": 3.96124, "gpu-sm-copy-bw/htod:0": 23.9685, "gpu-sm-copy-bw/htod:1": 23.967, "gpu-sm-copy-bw/htod:2": 19.9898, "gpu-sm-copy-bw/htod:3": 20.2848, "gpu-sm-copy-bw/htod:4": 20.3931, "gpu-sm-copy-bw/htod:5": 20.0888, "gpu-sm-copy-bw/htod:6": 23.9766, "gpu-sm-copy-bw/htod:7": 23.9792, "gpu-sm-copy-bw/htod:8": 23.9497, "gpu-sm-copy-bw/htod:9": 23.9438, "gpu-sm-copy-bw/htod:10": 20.0039, "gpu-sm-copy-bw/htod:11": 20.2469, "gpu-sm-copy-bw/htod:12": 20.4344, "gpu-sm-copy-bw/htod:13": 20.1005, "gpu-sm-copy-bw/htod:14": 23.9491, "gpu-sm-copy-bw/htod:15": 23.9898, "gpu-sm-copy-bw/htod:16": 23.962, "gpu-sm-copy-bw/htod:17": 23.97, "gpu-sm-copy-bw/htod:18": 19.987, "gpu-sm-copy-bw/htod:19": 20.1549, "gpu-sm-copy-bw/htod:20": 20.3931, "gpu-sm-copy-bw/htod:21": 20.1013, "gpu-sm-copy-bw/htod:22": 23.9865, "gpu-sm-copy-bw/htod:23": 23.972, "gpu-sm-copy-bw/htod:24": 23.9569, "gpu-sm-copy-bw/htod:25": 23.9762, "gpu-sm-copy-bw/htod:26": 19.92, "gpu-sm-copy-bw/htod:27": 20.2638, "gpu-sm-copy-bw/htod:28": 20.4419, "gpu-sm-copy-bw/htod:29": 20.1054, "gpu-sm-copy-bw/htod:30": 23.9752, "gpu-sm-copy-bw/htod:31": 23.9631, "ib-loopback/IB_write_512_Avg_0:0": 1492.89, "ib-loopback/IB_write_1024_Avg_0:0": 3224.92, "ib-loopback/IB_write_2048_Avg_0:0": 6714.5, "ib-loopback/IB_write_4096_Avg_0:0": 12871.93, "ib-loopback/IB_write_8192_Avg_0:0": 19990.78, "ib-loopback/IB_write_16384_Avg_0:0": 22172.25, "ib-loopback/IB_write_32768_Avg_0:0": 23073.19, "ib-loopback/IB_write_65536_Avg_0:0": 23527.09, "ib-loopback/IB_write_131072_Avg_0:0": 23805.92, "ib-loopback/IB_write_262144_Avg_0:0": 23380.51, "ib-loopback/IB_write_524288_Avg_0:0": 23856.63, "ib-loopback/IB_write_1048576_Avg_0:0": 23869.58, "ib-loopback/IB_write_2097152_Avg_0:0": 23885.7, "ib-loopback/IB_write_4194304_Avg_0:0": 23914.56, "ib-loopback/IB_write_8388608_Avg_0:0": 23935.21, "ib-loopback/IB_write_512_Avg_1:0": 1479.1, "ib-loopback/IB_write_1024_Avg_1:0": 3246.48, "ib-loopback/IB_write_2048_Avg_1:0": 6754.94, "ib-loopback/IB_write_4096_Avg_1:0": 13101.09, "ib-loopback/IB_write_8192_Avg_1:0": 19945.81, "ib-loopback/IB_write_16384_Avg_1:0": 22631.62, "ib-loopback/IB_write_32768_Avg_1:0": 23381.58, "ib-loopback/IB_write_65536_Avg_1:0": 23515.29, "ib-loopback/IB_write_131072_Avg_1:0": 23756.01, "ib-loopback/IB_write_262144_Avg_1:0": 23875.91, "ib-loopback/IB_write_524288_Avg_1:0": 23911.13, "ib-loopback/IB_write_1048576_Avg_1:0": 23935.42, "ib-loopback/IB_write_2097152_Avg_1:0": 23941.25, "ib-loopback/IB_write_4194304_Avg_1:0": 23922.06, "ib-loopback/IB_write_8388608_Avg_1:0": 23928.59, "ib-loopback/IB_write_512_Avg_2:0": 1505.29, "ib-loopback/IB_write_1024_Avg_2:0": 3215.97, "ib-loopback/IB_write_2048_Avg_2:0": 6745.49, "ib-loopback/IB_write_4096_Avg_2:0": 11548.88, "ib-loopback/IB_write_8192_Avg_2:0": 19432.15, "ib-loopback/IB_write_16384_Avg_2:0": 22765.51, "ib-loopback/IB_write_32768_Avg_2:0": 23235.07, "ib-loopback/IB_write_65536_Avg_2:0": 23620.08, "ib-loopback/IB_write_131072_Avg_2:0": 23759.08, "ib-loopback/IB_write_262144_Avg_2:0": 23859.82, "ib-loopback/IB_write_524288_Avg_2:0": 23775.01, "ib-loopback/IB_write_1048576_Avg_2:0": 23885.96, "ib-loopback/IB_write_2097152_Avg_2:0": 23894.73, "ib-loopback/IB_write_4194304_Avg_2:0": 23909.98, "ib-loopback/IB_write_8388608_Avg_2:0": 23927.21, "ib-loopback/IB_write_512_Avg_3:0": 1474.47, "ib-loopback/IB_write_1024_Avg_3:0": 3344.68, "ib-loopback/IB_write_2048_Avg_3:0": 6606.35, "ib-loopback/IB_write_4096_Avg_3:0": 12071.4, "ib-loopback/IB_write_8192_Avg_3:0": 18408.33, "ib-loopback/IB_write_16384_Avg_3:0": 20789.27, "ib-loopback/IB_write_32768_Avg_3:0": 22469.43, "ib-loopback/IB_write_65536_Avg_3:0": 22777.1, "ib-loopback/IB_write_131072_Avg_3:0": 23461.03, "ib-loopback/IB_write_262144_Avg_3:0": 23397.19, "ib-loopback/IB_write_524288_Avg_3:0": 23526.55, "ib-loopback/IB_write_1048576_Avg_3:0": 23854.76, "ib-loopback/IB_write_2097152_Avg_3:0": 23862.23, "ib-loopback/IB_write_4194304_Avg_3:0": 23931.15, "ib-loopback/IB_write_8388608_Avg_3:0": 23924.44, "ib-loopback/IB_write_512_Avg_4:0": 1523.33, "ib-loopback/IB_write_1024_Avg_4:0": 3233.23, "ib-loopback/IB_write_2048_Avg_4:0": 6792.88, "ib-loopback/IB_write_4096_Avg_4:0": 12616.05, "ib-loopback/IB_write_8192_Avg_4:0": 19324.05, "ib-loopback/IB_write_16384_Avg_4:0": 22082.51, "ib-loopback/IB_write_32768_Avg_4:0": 23294.23, "ib-loopback/IB_write_65536_Avg_4:0": 23546.22, "ib-loopback/IB_write_131072_Avg_4:0": 23727.91, "ib-loopback/IB_write_262144_Avg_4:0": 23843.93, "ib-loopback/IB_write_524288_Avg_4:0": 23905.96, "ib-loopback/IB_write_1048576_Avg_4:0": 23902.37, "ib-loopback/IB_write_2097152_Avg_4:0": 23921.03, "ib-loopback/IB_write_4194304_Avg_4:0": 23921.06, "ib-loopback/IB_write_8388608_Avg_4:0": 23922.4, "ib-loopback/IB_write_512_Avg_5:0": 1506.63, "ib-loopback/IB_write_1024_Avg_5:0": 3261.71, "ib-loopback/IB_write_2048_Avg_5:0": 6752.38, "ib-loopback/IB_write_4096_Avg_5:0": 13592.41, "ib-loopback/IB_write_8192_Avg_5:0": 19989.31, "ib-loopback/IB_write_16384_Avg_5:0": 22560.57, "ib-loopback/IB_write_32768_Avg_5:0": 23389.29, "ib-loopback/IB_write_65536_Avg_5:0": 23503.32, "ib-loopback/IB_write_131072_Avg_5:0": 23741.51, "ib-loopback/IB_write_262144_Avg_5:0": 23866.43, "ib-loopback/IB_write_524288_Avg_5:0": 23898.95, "ib-loopback/IB_write_1048576_Avg_5:0": 23876.36, "ib-loopback/IB_write_2097152_Avg_5:0": 23919.63, "ib-loopback/IB_write_4194304_Avg_5:0": 23924.68, "ib-loopback/IB_write_8388608_Avg_5:0": 23930.37, "ib-loopback/IB_write_512_Avg_6:0": 1467.69, "ib-loopback/IB_write_1024_Avg_6:0": 3157.04, "ib-loopback/IB_write_2048_Avg_6:0": 6494.61, "ib-loopback/IB_write_4096_Avg_6:0": 12883.51, "ib-loopback/IB_write_8192_Avg_6:0": 19207.67, "ib-loopback/IB_write_16384_Avg_6:0": 22519.39, "ib-loopback/IB_write_32768_Avg_6:0": 23323.46, "ib-loopback/IB_write_65536_Avg_6:0": 23523.6, "ib-loopback/IB_write_131072_Avg_6:0": 23626.67, "ib-loopback/IB_write_262144_Avg_6:0": 23836.99, "ib-loopback/IB_write_524288_Avg_6:0": 23904.51, "ib-loopback/IB_write_1048576_Avg_6:0": 23919.07, "ib-loopback/IB_write_2097152_Avg_6:0": 23943.82, "ib-loopback/IB_write_4194304_Avg_6:0": 23936.77, "ib-loopback/IB_write_8388608_Avg_6:0": 23941.57, "ib-loopback/IB_write_512_Avg_7:0": 1505.63, "ib-loopback/IB_write_1024_Avg_7:0": 3259.93, "ib-loopback/IB_write_2048_Avg_7:0": 6738.6, "ib-loopback/IB_write_4096_Avg_7:0": 13352.06, "ib-loopback/IB_write_8192_Avg_7:0": 19941.35, "ib-loopback/IB_write_16384_Avg_7:0": 22566.09, "ib-loopback/IB_write_32768_Avg_7:0": 23244.77, "ib-loopback/IB_write_65536_Avg_7:0": 23377.67, "ib-loopback/IB_write_131072_Avg_7:0": 23736.17, "ib-loopback/IB_write_262144_Avg_7:0": 23829.25, "ib-loopback/IB_write_524288_Avg_7:0": 23879.6, "ib-loopback/IB_write_1048576_Avg_7:0": 23895.1, "ib-loopback/IB_write_2097152_Avg_7:0": 23930.64, "ib-loopback/IB_write_4194304_Avg_7:0": 23845.63, "ib-loopback/IB_write_8388608_Avg_7:0": 23896.94, "kernel-launch/return_code":0, "kernel-launch/event_overhead:0": 0.1, "kernel-launch/event_overhead:1": 0.00595, "kernel-launch/event_overhead:2": 0.00557, "kernel-launch/event_overhead:3": 0.0055, "kernel-launch/event_overhead:4": 0.00592, "kernel-launch/event_overhead:5": 0.00589, "kernel-launch/event_overhead:6": 0.00572, "kernel-launch/event_overhead:7": 0.0059, "kernel-launch/wall_overhead:0": 0.01026, "kernel-launch/wall_overhead:1": 0.01026, "kernel-launch/wall_overhead:2": 0.01046, "kernel-launch/wall_overhead:3": 0.01049, "kernel-launch/wall_overhead:4": 0.01063, "kernel-launch/wall_overhead:5": 0.01006, "kernel-launch/wall_overhead:6": 0.01045, "kernel-launch/wall_overhead:7": 0.01071, "lstm_models/pytorch-lstm/steptime_train_float32": 48.07024518959224, "lstm_models/pytorch-lstm/throughput_train_float32": 4806.472441132788, "lstm_models/pytorch-lstm/steptime_train_float16": 25.95312986522913, "lstm_models/pytorch-lstm/throughput_train_float16": 9069.90809255883, "pytorch-matmul/nosharding": 34.64499759674072, "mem-bw/return_code":0, "mem-bw/H2D_Mem_BW:0": 25.6, "mem-bw/H2D_Mem_BW:1": 25.8, "mem-bw/H2D_Mem_BW:2": 26.0, "mem-bw/H2D_Mem_BW:3": 26.1, "mem-bw/H2D_Mem_BW:4": 26.2, "mem-bw/H2D_Mem_BW:5": 25.8, "mem-bw/H2D_Mem_BW:6": 25.3, "mem-bw/H2D_Mem_BW:7": 26.1, "mem-bw/D2H_Mem_BW:0": 24.3, "mem-bw/D2H_Mem_BW:1": 24.6, "mem-bw/D2H_Mem_BW:2": 24.5, "mem-bw/D2H_Mem_BW:3": 24.6, "mem-bw/D2H_Mem_BW:4": 24.3, "mem-bw/D2H_Mem_BW:5": 24.3, "mem-bw/D2H_Mem_BW:6": 23.9, "mem-bw/D2H_Mem_BW:7": 24.6, "mem-bw/D2D_Mem_BW:0": 1118.0, "mem-bw/D2D_Mem_BW:1": 1114.6, "mem-bw/D2D_Mem_BW:2": 1119.7, "mem-bw/D2D_Mem_BW:3": 1121.9, "mem-bw/D2D_Mem_BW:4": 1109.7, "mem-bw/D2D_Mem_BW:5": 1110.1, "mem-bw/D2D_Mem_BW:6": 1123.3, "mem-bw/D2D_Mem_BW:7": 1117.6, "nccl-bw/allreduce_8_busbw:0": 0.0, "nccl-bw/allreduce_8_algbw:0": 0.0, "nccl-bw/allreduce_8_time:0": 37.84, "nccl-bw/allreduce_16_busbw:0": 0.0, "nccl-bw/allreduce_16_algbw:0": 0.0, "nccl-bw/allreduce_16_time:0": 36.42, "nccl-bw/allreduce_32_busbw:0": 0.0, "nccl-bw/allreduce_32_algbw:0": 0.0, "nccl-bw/allreduce_32_time:0": 36.87, "nccl-bw/allreduce_64_busbw:0": 0.0, "nccl-bw/allreduce_64_algbw:0": 0.0, "nccl-bw/allreduce_64_time:0": 35.83, "nccl-bw/allreduce_128_busbw:0": 0.01, "nccl-bw/allreduce_128_algbw:0": 0.0, "nccl-bw/allreduce_128_time:0": 36.91, "nccl-bw/allreduce_256_busbw:0": 0.01, "nccl-bw/allreduce_256_algbw:0": 0.01, "nccl-bw/allreduce_256_time:0": 37.58, "nccl-bw/allreduce_512_busbw:0": 0.02, "nccl-bw/allreduce_512_algbw:0": 0.01, "nccl-bw/allreduce_512_time:0": 36.98, "nccl-bw/allreduce_1024_busbw:0": 0.05, "nccl-bw/allreduce_1024_algbw:0": 0.03, "nccl-bw/allreduce_1024_time:0": 36.93, "nccl-bw/allreduce_2048_busbw:0": 0.1, "nccl-bw/allreduce_2048_algbw:0": 0.06, "nccl-bw/allreduce_2048_time:0": 36.06, "nccl-bw/allreduce_4096_busbw:0": 0.19, "nccl-bw/allreduce_4096_algbw:0": 0.11, "nccl-bw/allreduce_4096_time:0": 37.2, "nccl-bw/allreduce_8192_busbw:0": 0.39, "nccl-bw/allreduce_8192_algbw:0": 0.22, "nccl-bw/allreduce_8192_time:0": 37.04, "nccl-bw/allreduce_16384_busbw:0": 0.77, "nccl-bw/allreduce_16384_algbw:0": 0.44, "nccl-bw/allreduce_16384_time:0": 37.46, "nccl-bw/allreduce_32768_busbw:0": 1.52, "nccl-bw/allreduce_32768_algbw:0": 0.87, "nccl-bw/allreduce_32768_time:0": 37.64, "nccl-bw/allreduce_65536_busbw:0": 3.0, "nccl-bw/allreduce_65536_algbw:0": 1.71, "nccl-bw/allreduce_65536_time:0": 38.22, "nccl-bw/allreduce_131072_busbw:0": 5.31, "nccl-bw/allreduce_131072_algbw:0": 3.04, "nccl-bw/allreduce_131072_time:0": 43.17, "nccl-bw/allreduce_262144_busbw:0": 9.5, "nccl-bw/allreduce_262144_algbw:0": 5.43, "nccl-bw/allreduce_262144_time:0": 48.29, "nccl-bw/allreduce_524288_busbw:0": 15.11, "nccl-bw/allreduce_524288_algbw:0": 8.64, "nccl-bw/allreduce_524288_time:0": 60.71, "nccl-bw/allreduce_1048576_busbw:0": 24.1, "nccl-bw/allreduce_1048576_algbw:0": 13.77, "nccl-bw/allreduce_1048576_time:0": 76.13, "nccl-bw/allreduce_2097152_busbw:0": 38.12, "nccl-bw/allreduce_2097152_algbw:0": 21.78, "nccl-bw/allreduce_2097152_time:0": 96.28, "nccl-bw/allreduce_4194304_busbw:0": 65.75, "nccl-bw/allreduce_4194304_algbw:0": 37.57, "nccl-bw/allreduce_4194304_time:0": 111.6, "nccl-bw/allreduce_8388608_busbw:0": 89.51, "nccl-bw/allreduce_8388608_algbw:0": 51.15, "nccl-bw/allreduce_8388608_time:0": 164.0, "nccl-bw/allreduce_16777216_busbw:0": 114.38, "nccl-bw/allreduce_16777216_algbw:0": 65.36, "nccl-bw/allreduce_16777216_time:0": 256.7, "nccl-bw/allreduce_33554432_busbw:0": 154.89, "nccl-bw/allreduce_33554432_algbw:0": 88.51, "nccl-bw/allreduce_33554432_time:0": 379.1, "nccl-bw/allreduce_67108864_busbw:0": 200.01, "nccl-bw/allreduce_67108864_algbw:0": 114.29, "nccl-bw/allreduce_67108864_time:0": 587.2, "nccl-bw/allreduce_134217728_busbw:0": 202.97, "nccl-bw/allreduce_134217728_algbw:0": 115.98, "nccl-bw/allreduce_134217728_time:0": 1157.2, "nccl-bw/allreduce_268435456_busbw:0": 221.82, "nccl-bw/allreduce_268435456_algbw:0": 126.75, "nccl-bw/allreduce_268435456_time:0": 2117.8, "nccl-bw/allreduce_536870912_busbw:0": 224.54, "nccl-bw/allreduce_536870912_algbw:0": 128.31, "nccl-bw/allreduce_536870912_time:0": 4184.2, "nccl-bw/allreduce_1073741824_busbw:0": 230.15, "nccl-bw/allreduce_1073741824_algbw:0": 131.51, "nccl-bw/allreduce_1073741824_time:0": 8164.5, "nccl-bw/allreduce_2147483648_busbw:0": 231.89, "nccl-bw/allreduce_2147483648_algbw:0": 132.51, "nccl-bw/allreduce_2147483648_time:0": 16207.0, "nccl-bw/allreduce_4294967296_busbw:0": 234.45, "nccl-bw/allreduce_4294967296_algbw:0": 133.97, "nccl-bw/allreduce_4294967296_time:0": 32059.0, "nccl-bw/allreduce_8589934592_busbw:0": 235.36, "nccl-bw/allreduce_8589934592_algbw:0": 134.49, "nccl-bw/allreduce_8589934592_time:0": 63870.0, "resnet_models/pytorch-resnet50/steptime_train_float32": 253.95522732287645, "resnet_models/pytorch-resnet50/throughput_train_float32": 760.3348099129964, "resnet_models/pytorch-resnet50/steptime_train_float16": 200.08606184273958, "resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430922575, "resnet_models/pytorch-resnet101/steptime_train_float32": 389.08605091273785, "resnet_models/pytorch-resnet101/throughput_train_float32": 496.11747409298965, "resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918143, "resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195069, "resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278314769, "resnet_models/pytorch-resnet152/throughput_train_float32": 352.07099543348215, "resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802225, "resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998153649, "pytorch-sharding-matmul/allreduce": 10.574411869049072, "pytorch-sharding-matmul/allgather": 10.084696769714355, "vgg_models/pytorch-vgg11/steptime_train_float32": 40.35283671692014, "vgg_models/pytorch-vgg11/throughput_train_float32": 796.3615936949874, "vgg_models/pytorch-vgg11/steptime_train_float16": 24.133514845743775, "vgg_models/pytorch-vgg11/throughput_train_float16": 1330.411361458461, "vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936026275, "vgg_models/pytorch-vgg13/throughput_train_float32": 580.234107444399, "vgg_models/pytorch-vgg13/steptime_train_float16": 33.35228993091732, "vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023901524, "vgg_models/pytorch-vgg16/steptime_train_float32": 65.22519944701344, "vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638875934, "vgg_models/pytorch-vgg16/steptime_train_float16": 39.25287735182792, "vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546147621, "vgg_models/pytorch-vgg19/steptime_train_float32": 74.93487105239183, "vgg_models/pytorch-vgg19/throughput_train_float32": 429.80921583106164, "vgg_models/pytorch-vgg19/steptime_train_float16": 45.20330624654889, "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377091} {"node": "sb-validation-02","bert_models/pytorch-bert-base/steptime_train_float32": 114.59167010616511,"bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591105, "bert_models/pytorch-bert-base/steptime_train_float16": 83.88951083179563, "bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582741963, "bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371913597, "bert_models/pytorch-bert-large/throughput_train_float32": 103.94876097417632, "bert_models/pytorch-bert-large/steptime_train_float16": 206.81141689419746, "bert_models/pytorch-bert-large/throughput_train_float16": 154.84089117113942, "pytorch-computation-communication-overlap/mul_cost:0": 44.182206214372854, "pytorch-computation-communication-overlap/mul_cost:1": 44.18221393893873, "pytorch-computation-communication-overlap/mul_cost:2": 43.970147078084665, "pytorch-computation-communication-overlap/mul_cost:3": 43.97014787559783, "pytorch-computation-communication-overlap/mul_cost:4": 43.97017793166924, "pytorch-computation-communication-overlap/mul_cost:5": 43.97015716062924, "pytorch-computation-communication-overlap/mul_cost:6": 43.97016519828867, "pytorch-computation-communication-overlap/mul_cost:7": 44.179544478538446, "pytorch-computation-communication-overlap/matmul_cost:0": 137.04773705558182, "pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344692856, "pytorch-computation-communication-overlap/matmul_cost:2": 137.04777220902997, "pytorch-computation-communication-overlap/matmul_cost:3": 137.04779697263803, "pytorch-computation-communication-overlap/matmul_cost:4": 137.04813674314664, "pytorch-computation-communication-overlap/matmul_cost:5": 137.04821988767435, "pytorch-computation-communication-overlap/matmul_cost:6": 137.04775322366913, "pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081606734, "densenet_models/pytorch-densenet169/steptime_train_float32": 150.64155543223023, "densenet_models/pytorch-densenet169/throughput_train_float32": 212.47017192425312, "densenet_models/pytorch-densenet169/steptime_train_float16": 145.30819572973996, "densenet_models/pytorch-densenet169/throughput_train_float16": 220.293233730535, "densenet_models/pytorch-densenet201/steptime_train_float32": 182.91117786429822, "densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232251, "densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596353, "densenet_models/pytorch-densenet201/throughput_train_float16": 181.54465736033902, "gemm-flops/FP64:0": 9031.23, "gemm-flops/FP64:1": 9040.85, "gemm-flops/FP64:2": 9010.56, "gemm-flops/FP64:3": 9041.26, "gemm-flops/FP64:4": 9039.19, "gemm-flops/FP64:5": 9015.69, "gemm-flops/FP64:6": 9022.19, "gemm-flops/FP64:7": 9030.2, "gemm-flops/FP32:0": 18362.1, "gemm-flops/FP32:1": 18375.6, "gemm-flops/FP32:2": 18314.9, "gemm-flops/FP32:3": 18375.6, "gemm-flops/FP32:4": 18368.6, "gemm-flops/FP32:5": 18347.1, "gemm-flops/FP32:6": 18247.4, "gemm-flops/FP32:7": 18318.4, "gemm-flops/FP16:0": 33878.0, "gemm-flops/FP16:1": 33911.1, "gemm-flops/FP16:2": 33769.3, "gemm-flops/FP16:3": 33909.9, "gemm-flops/FP16:4": 33896.5, "gemm-flops/FP16:5": 33798.1, "gemm-flops/FP16:6": 33647.3, "gemm-flops/FP16:7": 33764.8, "gemm-flops/FP64_TC:0": 18963.6, "gemm-flops/FP64_TC:1": 18924.2, "gemm-flops/FP64_TC:2": 18930.3, "gemm-flops/FP64_TC:3": 18971.9, "gemm-flops/FP64_TC:4": 18946.0, "gemm-flops/FP64_TC:5": 18945.0, "gemm-flops/FP64_TC:6": 18822.9, "gemm-flops/FP64_TC:7": 18911.1, "gemm-flops/TF32_TC:0": 127900.0, "gemm-flops/TF32_TC:1": 129094.0, "gemm-flops/TF32_TC:2": 127831.0, "gemm-flops/TF32_TC:3": 128709.0, "gemm-flops/TF32_TC:4": 127388.0, "gemm-flops/TF32_TC:5": 127861.0, "gemm-flops/TF32_TC:6": 128492.0, "gemm-flops/TF32_TC:7": 127720.0, "gemm-flops/BF16_TC:0": 264965.0, "gemm-flops/BF16_TC:1": 266638.0, "gemm-flops/BF16_TC:2": 263151.0, "gemm-flops/BF16_TC:3": 264752.0, "gemm-flops/BF16_TC:4": 263049.0, "gemm-flops/BF16_TC:5": 266605.0, "gemm-flops/BF16_TC:6": 267501.0, "gemm-flops/BF16_TC:7": 263880.0, "gemm-flops/FP16_TC:0": 279474.0, "gemm-flops/FP16_TC:1": 281256.0, "gemm-flops/FP16_TC:2": 277403.0, "gemm-flops/FP16_TC:3": 279147.0, "gemm-flops/FP16_TC:4": 277587.0, "gemm-flops/FP16_TC:5": 281537.0, "gemm-flops/FP16_TC:6": 282132.0, "gemm-flops/FP16_TC:7": 277788.0, "gemm-flops/INT8_TC:0": 475160.0, "gemm-flops/INT8_TC:1": 477725.0, "gemm-flops/INT8_TC:2": 471621.0, "gemm-flops/INT8_TC:3": 473716.0, "gemm-flops/INT8_TC:4": 472124.0, "gemm-flops/INT8_TC:5": 479972.0, "gemm-flops/INT8_TC:6": 481327.0, "gemm-flops/INT8_TC:7": 474710.0, "gemm-flops/INT4_TC:0": 970330.0, "gemm-flops/INT4_TC:1": 976837.0, "gemm-flops/INT4_TC:2": 966003.0, "gemm-flops/INT4_TC:3": 971315.0, "gemm-flops/INT4_TC:4": 964441.0, "gemm-flops/INT4_TC:5": 982461.0, "gemm-flops/INT4_TC:6": 979610.0, "gemm-flops/INT4_TC:7": 968359.0, "gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971835643, "gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.115454396866486, "gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742234692, "gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.139449941061585, "gpu-sm-copy-bw/dtoh:0": 3.91755, "gpu-sm-copy-bw/dtoh:1": 4.45414, "gpu-sm-copy-bw/dtoh:2": 1.26483, "gpu-sm-copy-bw/dtoh:3": 1.30041, "gpu-sm-copy-bw/dtoh:4": 1.31577, "gpu-sm-copy-bw/dtoh:5": 1.27968, "gpu-sm-copy-bw/dtoh:6": 4.47849, "gpu-sm-copy-bw/dtoh:7": 3.96231, "gpu-sm-copy-bw/dtoh:8": 3.91705, "gpu-sm-copy-bw/dtoh:9": 4.45487, "gpu-sm-copy-bw/dtoh:10": 1.26352, "gpu-sm-copy-bw/dtoh:11": 1.2999, "gpu-sm-copy-bw/dtoh:12": 1.31677, "gpu-sm-copy-bw/dtoh:13": 1.27885, "gpu-sm-copy-bw/dtoh:14": 4.47913, "gpu-sm-copy-bw/dtoh:15": 3.95893, "gpu-sm-copy-bw/dtoh:16": 3.91729, "gpu-sm-copy-bw/dtoh:17": 4.45627, "gpu-sm-copy-bw/dtoh:18": 1.26437, "gpu-sm-copy-bw/dtoh:19": 1.30144, "gpu-sm-copy-bw/dtoh:20": 1.31704, "gpu-sm-copy-bw/dtoh:21": 1.27857, "gpu-sm-copy-bw/dtoh:22": 4.47889, "gpu-sm-copy-bw/dtoh:23": 3.95984, "gpu-sm-copy-bw/dtoh:24": 3.92025, "gpu-sm-copy-bw/dtoh:25": 4.45423, "gpu-sm-copy-bw/dtoh:26": 1.26449, "gpu-sm-copy-bw/dtoh:27": 1.29954, "gpu-sm-copy-bw/dtoh:28": 1.31731, "gpu-sm-copy-bw/dtoh:29": 1.27916, "gpu-sm-copy-bw/dtoh:30": 4.4797, "gpu-sm-copy-bw/dtoh:31": 3.96124, "gpu-sm-copy-bw/htod:0": 23.9685, "gpu-sm-copy-bw/htod:1": 23.967, "gpu-sm-copy-bw/htod:2": 19.9898, "gpu-sm-copy-bw/htod:3": 20.2848, "gpu-sm-copy-bw/htod:4": 20.3931, "gpu-sm-copy-bw/htod:5": 20.0888, "gpu-sm-copy-bw/htod:6": 23.9766, "gpu-sm-copy-bw/htod:7": 23.9792, "gpu-sm-copy-bw/htod:8": 23.9497, "gpu-sm-copy-bw/htod:9": 23.9438, "gpu-sm-copy-bw/htod:10": 20.0039, "gpu-sm-copy-bw/htod:11": 20.2469, "gpu-sm-copy-bw/htod:12": 20.4344, "gpu-sm-copy-bw/htod:13": 20.1005, "gpu-sm-copy-bw/htod:14": 23.9491, "gpu-sm-copy-bw/htod:15": 23.9898, "gpu-sm-copy-bw/htod:16": 23.962, "gpu-sm-copy-bw/htod:17": 23.97, "gpu-sm-copy-bw/htod:18": 19.987, "gpu-sm-copy-bw/htod:19": 20.1549, "gpu-sm-copy-bw/htod:20": 20.3931, "gpu-sm-copy-bw/htod:21": 20.1013, "gpu-sm-copy-bw/htod:22": 23.9865, "gpu-sm-copy-bw/htod:23": 23.972, "gpu-sm-copy-bw/htod:24": 23.9569, "gpu-sm-copy-bw/htod:25": 23.9762, "gpu-sm-copy-bw/htod:26": 19.92, "gpu-sm-copy-bw/htod:27": 20.2638, "gpu-sm-copy-bw/htod:28": 20.4419, "gpu-sm-copy-bw/htod:29": 20.1054, "gpu-sm-copy-bw/htod:30": 23.9752, "gpu-sm-copy-bw/htod:31": 23.9631, "ib-loopback/IB_write_512_Avg_0:0": 1492.89, "ib-loopback/IB_write_1024_Avg_0:0": 3224.92, "ib-loopback/IB_write_2048_Avg_0:0": 6714.5, "ib-loopback/IB_write_4096_Avg_0:0": 12871.93, "ib-loopback/IB_write_8192_Avg_0:0": 19990.78, "ib-loopback/IB_write_16384_Avg_0:0": 22172.25, "ib-loopback/IB_write_32768_Avg_0:0": 23073.19, "ib-loopback/IB_write_65536_Avg_0:0": 23527.09, "ib-loopback/IB_write_131072_Avg_0:0": 23805.92, "ib-loopback/IB_write_262144_Avg_0:0": 23380.51, "ib-loopback/IB_write_524288_Avg_0:0": 23856.63, "ib-loopback/IB_write_1048576_Avg_0:0": 23869.58, "ib-loopback/IB_write_2097152_Avg_0:0": 23885.7, "ib-loopback/IB_write_4194304_Avg_0:0": 23914.56, "ib-loopback/IB_write_8388608_Avg_0:0": 23935.21, "ib-loopback/IB_write_512_Avg_1:0": 1479.1, "ib-loopback/IB_write_1024_Avg_1:0": 3246.48, "ib-loopback/IB_write_2048_Avg_1:0": 6754.94, "ib-loopback/IB_write_4096_Avg_1:0": 13101.09, "ib-loopback/IB_write_8192_Avg_1:0": 19945.81, "ib-loopback/IB_write_16384_Avg_1:0": 22631.62, "ib-loopback/IB_write_32768_Avg_1:0": 23381.58, "ib-loopback/IB_write_65536_Avg_1:0": 23515.29, "ib-loopback/IB_write_131072_Avg_1:0": 23756.01, "ib-loopback/IB_write_262144_Avg_1:0": 23875.91, "ib-loopback/IB_write_524288_Avg_1:0": 23911.13, "ib-loopback/IB_write_1048576_Avg_1:0": 23935.42, "ib-loopback/IB_write_2097152_Avg_1:0": 23941.25, "ib-loopback/IB_write_4194304_Avg_1:0": 23922.06, "ib-loopback/IB_write_8388608_Avg_1:0": 23928.59, "ib-loopback/IB_write_512_Avg_2:0": 1505.29, "ib-loopback/IB_write_1024_Avg_2:0": 3215.97, "ib-loopback/IB_write_2048_Avg_2:0": 6745.49, "ib-loopback/IB_write_4096_Avg_2:0": 11548.88, "ib-loopback/IB_write_8192_Avg_2:0": 19432.15, "ib-loopback/IB_write_16384_Avg_2:0": 22765.51, "ib-loopback/IB_write_32768_Avg_2:0": 23235.07, "ib-loopback/IB_write_65536_Avg_2:0": 23620.08, "ib-loopback/IB_write_131072_Avg_2:0": 23759.08, "ib-loopback/IB_write_262144_Avg_2:0": 23859.82, "ib-loopback/IB_write_524288_Avg_2:0": 23775.01, "ib-loopback/IB_write_1048576_Avg_2:0": 23885.96, "ib-loopback/IB_write_2097152_Avg_2:0": 23894.73, "ib-loopback/IB_write_4194304_Avg_2:0": 23909.98, "ib-loopback/IB_write_8388608_Avg_2:0": 23927.21, "ib-loopback/IB_write_512_Avg_3:0": 1474.47, "ib-loopback/IB_write_1024_Avg_3:0": 3344.68, "ib-loopback/IB_write_2048_Avg_3:0": 6606.35, "ib-loopback/IB_write_4096_Avg_3:0": 12071.4, "ib-loopback/IB_write_8192_Avg_3:0": 18408.33, "ib-loopback/IB_write_16384_Avg_3:0": 20789.27, "ib-loopback/IB_write_32768_Avg_3:0": 22469.43, "ib-loopback/IB_write_65536_Avg_3:0": 22777.1, "ib-loopback/IB_write_131072_Avg_3:0": 23461.03, "ib-loopback/IB_write_262144_Avg_3:0": 23397.19, "ib-loopback/IB_write_524288_Avg_3:0": 23526.55, "ib-loopback/IB_write_1048576_Avg_3:0": 23854.76, "ib-loopback/IB_write_2097152_Avg_3:0": 23862.23, "ib-loopback/IB_write_4194304_Avg_3:0": 23931.15, "ib-loopback/IB_write_8388608_Avg_3:0": 23924.44, "ib-loopback/IB_write_512_Avg_4:0": 1523.33, "ib-loopback/IB_write_1024_Avg_4:0": 3233.23, "ib-loopback/IB_write_2048_Avg_4:0": 6792.88, "ib-loopback/IB_write_4096_Avg_4:0": 12616.05, "ib-loopback/IB_write_8192_Avg_4:0": 19324.05, "ib-loopback/IB_write_16384_Avg_4:0": 22082.51, "ib-loopback/IB_write_32768_Avg_4:0": 23294.23, "ib-loopback/IB_write_65536_Avg_4:0": 23546.22, "ib-loopback/IB_write_131072_Avg_4:0": 23727.91, "ib-loopback/IB_write_262144_Avg_4:0": 23843.93, "ib-loopback/IB_write_524288_Avg_4:0": 23905.96, "ib-loopback/IB_write_1048576_Avg_4:0": 23902.37, "ib-loopback/IB_write_2097152_Avg_4:0": 23921.03, "ib-loopback/IB_write_4194304_Avg_4:0": 23921.06, "ib-loopback/IB_write_8388608_Avg_4:0": 23922.4, "ib-loopback/IB_write_512_Avg_5:0": 1506.63, "ib-loopback/IB_write_1024_Avg_5:0": 3261.71, "ib-loopback/IB_write_2048_Avg_5:0": 6752.38, "ib-loopback/IB_write_4096_Avg_5:0": 13592.41, "ib-loopback/IB_write_8192_Avg_5:0": 19989.31, "ib-loopback/IB_write_16384_Avg_5:0": 22560.57, "ib-loopback/IB_write_32768_Avg_5:0": 23389.29, "ib-loopback/IB_write_65536_Avg_5:0": 23503.32, "ib-loopback/IB_write_131072_Avg_5:0": 23741.51, "ib-loopback/IB_write_262144_Avg_5:0": 23866.43, "ib-loopback/IB_write_524288_Avg_5:0": 23898.95, "ib-loopback/IB_write_1048576_Avg_5:0": 23876.36, "ib-loopback/IB_write_2097152_Avg_5:0": 23919.63, "ib-loopback/IB_write_4194304_Avg_5:0": 23924.68, "ib-loopback/IB_write_8388608_Avg_5:0": 23930.37, "ib-loopback/IB_write_512_Avg_6:0": 1467.69, "ib-loopback/IB_write_1024_Avg_6:0": 3157.04, "ib-loopback/IB_write_2048_Avg_6:0": 6494.61, "ib-loopback/IB_write_4096_Avg_6:0": 12883.51, "ib-loopback/IB_write_8192_Avg_6:0": 19207.67, "ib-loopback/IB_write_16384_Avg_6:0": 22519.39, "ib-loopback/IB_write_32768_Avg_6:0": 23323.46, "ib-loopback/IB_write_65536_Avg_6:0": 23523.6, "ib-loopback/IB_write_131072_Avg_6:0": 23626.67, "ib-loopback/IB_write_262144_Avg_6:0": 23836.99, "ib-loopback/IB_write_524288_Avg_6:0": 23904.51, "ib-loopback/IB_write_1048576_Avg_6:0": 23919.07, "ib-loopback/IB_write_2097152_Avg_6:0": 23943.82, "ib-loopback/IB_write_4194304_Avg_6:0": 23936.77, "ib-loopback/IB_write_8388608_Avg_6:0": 23941.57, "ib-loopback/IB_write_512_Avg_7:0": 1505.63, "ib-loopback/IB_write_1024_Avg_7:0": 3259.93, "ib-loopback/IB_write_2048_Avg_7:0": 6738.6, "ib-loopback/IB_write_4096_Avg_7:0": 13352.06, "ib-loopback/IB_write_8192_Avg_7:0": 19941.35, "ib-loopback/IB_write_16384_Avg_7:0": 22566.09, "ib-loopback/IB_write_32768_Avg_7:0": 23244.77, "ib-loopback/IB_write_65536_Avg_7:0": 23377.67, "ib-loopback/IB_write_131072_Avg_7:0": 23736.17, "ib-loopback/IB_write_262144_Avg_7:0": 23829.25, "ib-loopback/IB_write_524288_Avg_7:0": 23879.6, "ib-loopback/IB_write_1048576_Avg_7:0": 23895.1, "ib-loopback/IB_write_2097152_Avg_7:0": 23930.64, "ib-loopback/IB_write_4194304_Avg_7:0": 23845.63, "ib-loopback/IB_write_8388608_Avg_7:0": 23896.94, "kernel-launch/return_code":0, "kernel-launch/event_overhead:0": 0.00595, "kernel-launch/event_overhead:1": 0.00595, "kernel-launch/event_overhead:2": 0.00557, "kernel-launch/event_overhead:3": 0.0055, "kernel-launch/event_overhead:4": 0.00592, "kernel-launch/event_overhead:5": 0.00589, "kernel-launch/event_overhead:6": 0.00572, "kernel-launch/event_overhead:7": 0.0059, "kernel-launch/wall_overhead:0": 0.01026, "kernel-launch/wall_overhead:1": 0.01026, "kernel-launch/wall_overhead:2": 0.01046, "kernel-launch/wall_overhead:3": 0.01049, "kernel-launch/wall_overhead:4": 0.01063, "kernel-launch/wall_overhead:5": 0.01006, "kernel-launch/wall_overhead:6": 0.01045, "kernel-launch/wall_overhead:7": 0.01071, "lstm_models/pytorch-lstm/steptime_train_float32": 48.07024518959224, "lstm_models/pytorch-lstm/throughput_train_float32": 4806.472441132788, "lstm_models/pytorch-lstm/steptime_train_float16": 25.95312986522913, "lstm_models/pytorch-lstm/throughput_train_float16": 9069.90809255883, "pytorch-matmul/nosharding": 34.64499759674072, "mem-bw/return_code":0, "mem-bw/H2D_Mem_BW:0": 25.6, "mem-bw/H2D_Mem_BW:1": 25.8, "mem-bw/H2D_Mem_BW:2": 26.0, "mem-bw/H2D_Mem_BW:3": 26.1, "mem-bw/H2D_Mem_BW:4": 26.2, "mem-bw/H2D_Mem_BW:5": 25.8, "mem-bw/H2D_Mem_BW:6": 25.3, "mem-bw/H2D_Mem_BW:7": 26.1, "mem-bw/D2H_Mem_BW:0": 24.3, "mem-bw/D2H_Mem_BW:1": 24.6, "mem-bw/D2H_Mem_BW:2": 24.5, "mem-bw/D2H_Mem_BW:3": 24.6, "mem-bw/D2H_Mem_BW:4": 24.3, "mem-bw/D2H_Mem_BW:5": 24.3, "mem-bw/D2H_Mem_BW:6": 23.9, "mem-bw/D2H_Mem_BW:7": 24.6, "mem-bw/D2D_Mem_BW:0": 1118.0, "mem-bw/D2D_Mem_BW:1": 1114.6, "mem-bw/D2D_Mem_BW:2": 1119.7, "mem-bw/D2D_Mem_BW:3": 1121.9, "mem-bw/D2D_Mem_BW:4": 1109.7, "mem-bw/D2D_Mem_BW:5": 1110.1, "mem-bw/D2D_Mem_BW:6": 1123.3, "mem-bw/D2D_Mem_BW:7": 1117.6, "nccl-bw/allreduce_8_busbw:0": 0.0, "nccl-bw/allreduce_8_algbw:0": 0.0, "nccl-bw/allreduce_8_time:0": 37.84, "nccl-bw/allreduce_16_busbw:0": 0.0, "nccl-bw/allreduce_16_algbw:0": 0.0, "nccl-bw/allreduce_16_time:0": 36.42, "nccl-bw/allreduce_32_busbw:0": 0.0, "nccl-bw/allreduce_32_algbw:0": 0.0, "nccl-bw/allreduce_32_time:0": 36.87, "nccl-bw/allreduce_64_busbw:0": 0.0, "nccl-bw/allreduce_64_algbw:0": 0.0, "nccl-bw/allreduce_64_time:0": 35.83, "nccl-bw/allreduce_128_busbw:0": 0.01, "nccl-bw/allreduce_128_algbw:0": 0.0, "nccl-bw/allreduce_128_time:0": 36.91, "nccl-bw/allreduce_256_busbw:0": 0.01, "nccl-bw/allreduce_256_algbw:0": 0.01, "nccl-bw/allreduce_256_time:0": 37.58, "nccl-bw/allreduce_512_busbw:0": 0.02, "nccl-bw/allreduce_512_algbw:0": 0.01, "nccl-bw/allreduce_512_time:0": 36.98, "nccl-bw/allreduce_1024_busbw:0": 0.05, "nccl-bw/allreduce_1024_algbw:0": 0.03, "nccl-bw/allreduce_1024_time:0": 36.93, "nccl-bw/allreduce_2048_busbw:0": 0.1, "nccl-bw/allreduce_2048_algbw:0": 0.06, "nccl-bw/allreduce_2048_time:0": 36.06, "nccl-bw/allreduce_4096_busbw:0": 0.19, "nccl-bw/allreduce_4096_algbw:0": 0.11, "nccl-bw/allreduce_4096_time:0": 37.2, "nccl-bw/allreduce_8192_busbw:0": 0.39, "nccl-bw/allreduce_8192_algbw:0": 0.22, "nccl-bw/allreduce_8192_time:0": 37.04, "nccl-bw/allreduce_16384_busbw:0": 0.77, "nccl-bw/allreduce_16384_algbw:0": 0.44, "nccl-bw/allreduce_16384_time:0": 37.46, "nccl-bw/allreduce_32768_busbw:0": 1.52, "nccl-bw/allreduce_32768_algbw:0": 0.87, "nccl-bw/allreduce_32768_time:0": 37.64, "nccl-bw/allreduce_65536_busbw:0": 3.0, "nccl-bw/allreduce_65536_algbw:0": 1.71, "nccl-bw/allreduce_65536_time:0": 38.22, "nccl-bw/allreduce_131072_busbw:0": 5.31, "nccl-bw/allreduce_131072_algbw:0": 3.04, "nccl-bw/allreduce_131072_time:0": 43.17, "nccl-bw/allreduce_262144_busbw:0": 9.5, "nccl-bw/allreduce_262144_algbw:0": 5.43, "nccl-bw/allreduce_262144_time:0": 48.29, "nccl-bw/allreduce_524288_busbw:0": 15.11, "nccl-bw/allreduce_524288_algbw:0": 8.64, "nccl-bw/allreduce_524288_time:0": 60.71, "nccl-bw/allreduce_1048576_busbw:0": 24.1, "nccl-bw/allreduce_1048576_algbw:0": 13.77, "nccl-bw/allreduce_1048576_time:0": 76.13, "nccl-bw/allreduce_2097152_busbw:0": 38.12, "nccl-bw/allreduce_2097152_algbw:0": 21.78, "nccl-bw/allreduce_2097152_time:0": 96.28, "nccl-bw/allreduce_4194304_busbw:0": 65.75, "nccl-bw/allreduce_4194304_algbw:0": 37.57, "nccl-bw/allreduce_4194304_time:0": 111.6, "nccl-bw/allreduce_8388608_busbw:0": 89.51, "nccl-bw/allreduce_8388608_algbw:0": 51.15, "nccl-bw/allreduce_8388608_time:0": 164.0, "nccl-bw/allreduce_16777216_busbw:0": 114.38, "nccl-bw/allreduce_16777216_algbw:0": 65.36, "nccl-bw/allreduce_16777216_time:0": 256.7, "nccl-bw/allreduce_33554432_busbw:0": 154.89, "nccl-bw/allreduce_33554432_algbw:0": 88.51, "nccl-bw/allreduce_33554432_time:0": 379.1, "nccl-bw/allreduce_67108864_busbw:0": 200.01, "nccl-bw/allreduce_67108864_algbw:0": 114.29, "nccl-bw/allreduce_67108864_time:0": 587.2, "nccl-bw/allreduce_134217728_busbw:0": 202.97, "nccl-bw/allreduce_134217728_algbw:0": 115.98, "nccl-bw/allreduce_134217728_time:0": 1157.2, "nccl-bw/allreduce_268435456_busbw:0": 221.82, "nccl-bw/allreduce_268435456_algbw:0": 126.75, "nccl-bw/allreduce_268435456_time:0": 2117.8, "nccl-bw/allreduce_536870912_busbw:0": 224.54, "nccl-bw/allreduce_536870912_algbw:0": 128.31, "nccl-bw/allreduce_536870912_time:0": 4184.2, "nccl-bw/allreduce_1073741824_busbw:0": 230.15, "nccl-bw/allreduce_1073741824_algbw:0": 131.51, "nccl-bw/allreduce_1073741824_time:0": 8164.5, "nccl-bw/allreduce_2147483648_busbw:0": 231.89, "nccl-bw/allreduce_2147483648_algbw:0": 132.51, "nccl-bw/allreduce_2147483648_time:0": 16207.0, "nccl-bw/allreduce_4294967296_busbw:0": 234.45, "nccl-bw/allreduce_4294967296_algbw:0": 133.97, "nccl-bw/allreduce_4294967296_time:0": 32059.0, "nccl-bw/allreduce_8589934592_busbw:0": 235.36, "nccl-bw/allreduce_8589934592_algbw:0": 134.49, "nccl-bw/allreduce_8589934592_time:0": 63870.0, "resnet_models/pytorch-resnet50/steptime_train_float32": 253.95522732287645, "resnet_models/pytorch-resnet50/throughput_train_float32": 760.3348099129964, "resnet_models/pytorch-resnet50/steptime_train_float16": 200.08606184273958, "resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430922575, "resnet_models/pytorch-resnet101/steptime_train_float32": 389.08605091273785, "resnet_models/pytorch-resnet101/throughput_train_float32": 496.11747409298965, "resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918143, "resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195069, "resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278314769, "resnet_models/pytorch-resnet152/throughput_train_float32": 352.07099543348215, "resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802225, "resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998153649, "pytorch-sharding-matmul/allreduce": 10.574411869049072, "pytorch-sharding-matmul/allgather": 10.084696769714355, "vgg_models/pytorch-vgg11/steptime_train_float32": 40.35283671692014, "vgg_models/pytorch-vgg11/throughput_train_float32": 796.3615936949874, "vgg_models/pytorch-vgg11/steptime_train_float16": 24.133514845743775, "vgg_models/pytorch-vgg11/throughput_train_float16": 1330.411361458461, "vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936026275, "vgg_models/pytorch-vgg13/throughput_train_float32": 580.234107444399, "vgg_models/pytorch-vgg13/steptime_train_float16": 33.35228993091732, "vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023901524, "vgg_models/pytorch-vgg16/steptime_train_float32": 65.22519944701344, "vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638875934, "vgg_models/pytorch-vgg16/steptime_train_float16": 39.25287735182792, "vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546147621, "vgg_models/pytorch-vgg19/steptime_train_float32": 74.93487105239183, "vgg_models/pytorch-vgg19/throughput_train_float32": 429.80921583106164, "vgg_models/pytorch-vgg19/steptime_train_float16": 45.20330624654889, "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377091} {"node": "sb-validation-03","bert_models/pytorch-bert-base/steptime_train_float32": 114.59167010616511,"bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591105, "bert_models/pytorch-bert-base/steptime_train_float16": 83.88951083179563, "bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582741963, "bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371913597, "bert_models/pytorch-bert-large/throughput_train_float32": 103.94876097417632, "bert_models/pytorch-bert-large/steptime_train_float16": 206.81141689419746, "bert_models/pytorch-bert-large/throughput_train_float16": 154.84089117113942, "pytorch-computation-communication-overlap/mul_cost:0": 44.182206214372854, "pytorch-computation-communication-overlap/mul_cost:1": 44.18221393893873, "pytorch-computation-communication-overlap/mul_cost:2": 43.970147078084665, "pytorch-computation-communication-overlap/mul_cost:3": 43.97014787559783, "pytorch-computation-communication-overlap/mul_cost:4": 43.97017793166924, "pytorch-computation-communication-overlap/mul_cost:5": 43.97015716062924, "pytorch-computation-communication-overlap/mul_cost:6": 43.97016519828867, "pytorch-computation-communication-overlap/mul_cost:7": 44.179544478538446, "pytorch-computation-communication-overlap/matmul_cost:0": 137.04773705558182, "pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344692856, "pytorch-computation-communication-overlap/matmul_cost:2": 137.04777220902997, "pytorch-computation-communication-overlap/matmul_cost:3": 137.04779697263803, "pytorch-computation-communication-overlap/matmul_cost:4": 137.04813674314664, "pytorch-computation-communication-overlap/matmul_cost:5": 137.04821988767435, "pytorch-computation-communication-overlap/matmul_cost:6": 137.04775322366913, "pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081606734, "densenet_models/pytorch-densenet169/steptime_train_float32": 150.64155543223023, "densenet_models/pytorch-densenet169/throughput_train_float32": 212.47017192425312, "densenet_models/pytorch-densenet169/steptime_train_float16": 145.30819572973996, "densenet_models/pytorch-densenet169/throughput_train_float16": 220.293233730535, "densenet_models/pytorch-densenet201/steptime_train_float32": 182.91117786429822, "densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232251, "densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596353, "densenet_models/pytorch-densenet201/throughput_train_float16": 181.54465736033902, "gemm-flops/FP64:0": 9031.23, "gemm-flops/FP64:1": 9040.85, "gemm-flops/FP64:2": 9010.56, "gemm-flops/FP64:3": 9041.26, "gemm-flops/FP64:4": 9039.19, "gemm-flops/FP64:5": 9015.69, "gemm-flops/FP64:6": 9022.19, "gemm-flops/FP64:7": 9030.2, "gemm-flops/FP32:0": 18362.1, "gemm-flops/FP32:1": 18375.6, "gemm-flops/FP32:2": 18314.9, "gemm-flops/FP32:3": 18375.6, "gemm-flops/FP32:4": 18368.6, "gemm-flops/FP32:5": 18347.1, "gemm-flops/FP32:6": 18247.4, "gemm-flops/FP32:7": 18318.4, "gemm-flops/FP16:0": 33878.0, "gemm-flops/FP16:1": 33911.1, "gemm-flops/FP16:2": 33769.3, "gemm-flops/FP16:3": 33909.9, "gemm-flops/FP16:4": 33896.5, "gemm-flops/FP16:5": 33798.1, "gemm-flops/FP16:6": 33647.3, "gemm-flops/FP16:7": 33764.8, "gemm-flops/FP64_TC:0": 18963.6, "gemm-flops/FP64_TC:1": 18924.2, "gemm-flops/FP64_TC:2": 18930.3, "gemm-flops/FP64_TC:3": 18971.9, "gemm-flops/FP64_TC:4": 18946.0, "gemm-flops/FP64_TC:5": 18945.0, "gemm-flops/FP64_TC:6": 18822.9, "gemm-flops/FP64_TC:7": 18911.1, "gemm-flops/TF32_TC:0": 127900.0, "gemm-flops/TF32_TC:1": 129094.0, "gemm-flops/TF32_TC:2": 127831.0, "gemm-flops/TF32_TC:3": 128709.0, "gemm-flops/TF32_TC:4": 127388.0, "gemm-flops/TF32_TC:5": 127861.0, "gemm-flops/TF32_TC:6": 128492.0, "gemm-flops/TF32_TC:7": 127720.0, "gemm-flops/BF16_TC:0": 264965.0, "gemm-flops/BF16_TC:1": 266638.0, "gemm-flops/BF16_TC:2": 263151.0, "gemm-flops/BF16_TC:3": 264752.0, "gemm-flops/BF16_TC:4": 263049.0, "gemm-flops/BF16_TC:5": 266605.0, "gemm-flops/BF16_TC:6": 267501.0, "gemm-flops/BF16_TC:7": 263880.0, "gemm-flops/FP16_TC:0": 279474.0, "gemm-flops/FP16_TC:1": 281256.0, "gemm-flops/FP16_TC:2": 277403.0, "gemm-flops/FP16_TC:3": 279147.0, "gemm-flops/FP16_TC:4": 277587.0, "gemm-flops/FP16_TC:5": 281537.0, "gemm-flops/FP16_TC:6": 282132.0, "gemm-flops/FP16_TC:7": 277788.0, "gemm-flops/INT8_TC:0": 475160.0, "gemm-flops/INT8_TC:1": 477725.0, "gemm-flops/INT8_TC:2": 471621.0, "gemm-flops/INT8_TC:3": 473716.0, "gemm-flops/INT8_TC:4": 472124.0, "gemm-flops/INT8_TC:5": 479972.0, "gemm-flops/INT8_TC:6": 481327.0, "gemm-flops/INT8_TC:7": 474710.0, "gemm-flops/INT4_TC:0": 970330.0, "gemm-flops/INT4_TC:1": 976837.0, "gemm-flops/INT4_TC:2": 966003.0, "gemm-flops/INT4_TC:3": 971315.0, "gemm-flops/INT4_TC:4": 964441.0, "gemm-flops/INT4_TC:5": 982461.0, "gemm-flops/INT4_TC:6": 979610.0, "gemm-flops/INT4_TC:7": 968359.0, "gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971835643, "gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.115454396866486, "gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742234692, "gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.139449941061585, "gpu-sm-copy-bw/dtoh:0": 3.91755, "gpu-sm-copy-bw/dtoh:1": 4.45414, "gpu-sm-copy-bw/dtoh:2": 1.26483, "gpu-sm-copy-bw/dtoh:3": 1.30041, "gpu-sm-copy-bw/dtoh:4": 1.31577, "gpu-sm-copy-bw/dtoh:5": 1.27968, "gpu-sm-copy-bw/dtoh:6": 4.47849, "gpu-sm-copy-bw/dtoh:7": 3.96231, "gpu-sm-copy-bw/dtoh:8": 3.91705, "gpu-sm-copy-bw/dtoh:9": 4.45487, "gpu-sm-copy-bw/dtoh:10": 1.26352, "gpu-sm-copy-bw/dtoh:11": 1.2999, "gpu-sm-copy-bw/dtoh:12": 1.31677, "gpu-sm-copy-bw/dtoh:13": 1.27885, "gpu-sm-copy-bw/dtoh:14": 4.47913, "gpu-sm-copy-bw/dtoh:15": 3.95893, "gpu-sm-copy-bw/dtoh:16": 3.91729, "gpu-sm-copy-bw/dtoh:17": 4.45627, "gpu-sm-copy-bw/dtoh:18": 1.26437, "gpu-sm-copy-bw/dtoh:19": 1.30144, "gpu-sm-copy-bw/dtoh:20": 1.31704, "gpu-sm-copy-bw/dtoh:21": 1.27857, "gpu-sm-copy-bw/dtoh:22": 4.47889, "gpu-sm-copy-bw/dtoh:23": 3.95984, "gpu-sm-copy-bw/dtoh:24": 3.92025, "gpu-sm-copy-bw/dtoh:25": 4.45423, "gpu-sm-copy-bw/dtoh:26": 1.26449, "gpu-sm-copy-bw/dtoh:27": 1.29954, "gpu-sm-copy-bw/dtoh:28": 1.31731, "gpu-sm-copy-bw/dtoh:29": 1.27916, "gpu-sm-copy-bw/dtoh:30": 4.4797, "gpu-sm-copy-bw/dtoh:31": 3.96124, "gpu-sm-copy-bw/htod:0": 23.9685, "gpu-sm-copy-bw/htod:1": 23.967, "gpu-sm-copy-bw/htod:2": 19.9898, "gpu-sm-copy-bw/htod:3": 20.2848, "gpu-sm-copy-bw/htod:4": 20.3931, "gpu-sm-copy-bw/htod:5": 20.0888, "gpu-sm-copy-bw/htod:6": 23.9766, "gpu-sm-copy-bw/htod:7": 23.9792, "gpu-sm-copy-bw/htod:8": 23.9497, "gpu-sm-copy-bw/htod:9": 23.9438, "gpu-sm-copy-bw/htod:10": 20.0039, "gpu-sm-copy-bw/htod:11": 20.2469, "gpu-sm-copy-bw/htod:12": 20.4344, "gpu-sm-copy-bw/htod:13": 20.1005, "gpu-sm-copy-bw/htod:14": 23.9491, "gpu-sm-copy-bw/htod:15": 23.9898, "gpu-sm-copy-bw/htod:16": 23.962, "gpu-sm-copy-bw/htod:17": 23.97, "gpu-sm-copy-bw/htod:18": 19.987, "gpu-sm-copy-bw/htod:19": 20.1549, "gpu-sm-copy-bw/htod:20": 20.3931, "gpu-sm-copy-bw/htod:21": 20.1013, "gpu-sm-copy-bw/htod:22": 23.9865, "gpu-sm-copy-bw/htod:23": 23.972, "gpu-sm-copy-bw/htod:24": 23.9569, "gpu-sm-copy-bw/htod:25": 23.9762, "gpu-sm-copy-bw/htod:26": 19.92, "gpu-sm-copy-bw/htod:27": 20.2638, "gpu-sm-copy-bw/htod:28": 20.4419, "gpu-sm-copy-bw/htod:29": 20.1054, "gpu-sm-copy-bw/htod:30": 23.9752, "gpu-sm-copy-bw/htod:31": 23.9631, "ib-loopback/IB_write_512_Avg_0:0": 1492.89, "ib-loopback/IB_write_1024_Avg_0:0": 3224.92, "ib-loopback/IB_write_2048_Avg_0:0": 6714.5, "ib-loopback/IB_write_4096_Avg_0:0": 12871.93, "ib-loopback/IB_write_8192_Avg_0:0": 19990.78, "ib-loopback/IB_write_16384_Avg_0:0": 22172.25, "ib-loopback/IB_write_32768_Avg_0:0": 23073.19, "ib-loopback/IB_write_65536_Avg_0:0": 23527.09, "ib-loopback/IB_write_131072_Avg_0:0": 23805.92, "ib-loopback/IB_write_262144_Avg_0:0": 23380.51, "ib-loopback/IB_write_524288_Avg_0:0": 23856.63, "ib-loopback/IB_write_1048576_Avg_0:0": 23869.58, "ib-loopback/IB_write_2097152_Avg_0:0": 23885.7, "ib-loopback/IB_write_4194304_Avg_0:0": 23914.56, "ib-loopback/IB_write_8388608_Avg_0:0": 23935.21, "ib-loopback/IB_write_512_Avg_1:0": 1479.1, "ib-loopback/IB_write_1024_Avg_1:0": 3246.48, "ib-loopback/IB_write_2048_Avg_1:0": 6754.94, "ib-loopback/IB_write_4096_Avg_1:0": 13101.09, "ib-loopback/IB_write_8192_Avg_1:0": 19945.81, "ib-loopback/IB_write_16384_Avg_1:0": 22631.62, "ib-loopback/IB_write_32768_Avg_1:0": 23381.58, "ib-loopback/IB_write_65536_Avg_1:0": 23515.29, "ib-loopback/IB_write_131072_Avg_1:0": 23756.01, "ib-loopback/IB_write_262144_Avg_1:0": 23875.91, "ib-loopback/IB_write_524288_Avg_1:0": 23911.13, "ib-loopback/IB_write_1048576_Avg_1:0": 23935.42, "ib-loopback/IB_write_2097152_Avg_1:0": 23941.25, "ib-loopback/IB_write_4194304_Avg_1:0": 23922.06, "ib-loopback/IB_write_8388608_Avg_1:0": 23928.59, "ib-loopback/IB_write_512_Avg_2:0": 1505.29, "ib-loopback/IB_write_1024_Avg_2:0": 3215.97, "ib-loopback/IB_write_2048_Avg_2:0": 6745.49, "ib-loopback/IB_write_4096_Avg_2:0": 11548.88, "ib-loopback/IB_write_8192_Avg_2:0": 19432.15, "ib-loopback/IB_write_16384_Avg_2:0": 22765.51, "ib-loopback/IB_write_32768_Avg_2:0": 23235.07, "ib-loopback/IB_write_65536_Avg_2:0": 23620.08, "ib-loopback/IB_write_131072_Avg_2:0": 23759.08, "ib-loopback/IB_write_262144_Avg_2:0": 23859.82, "ib-loopback/IB_write_524288_Avg_2:0": 23775.01, "ib-loopback/IB_write_1048576_Avg_2:0": 23885.96, "ib-loopback/IB_write_2097152_Avg_2:0": 23894.73, "ib-loopback/IB_write_4194304_Avg_2:0": 23909.98, "ib-loopback/IB_write_8388608_Avg_2:0": 23927.21, "ib-loopback/IB_write_512_Avg_3:0": 1474.47, "ib-loopback/IB_write_1024_Avg_3:0": 3344.68, "ib-loopback/IB_write_2048_Avg_3:0": 6606.35, "ib-loopback/IB_write_4096_Avg_3:0": 12071.4, "ib-loopback/IB_write_8192_Avg_3:0": 18408.33, "ib-loopback/IB_write_16384_Avg_3:0": 20789.27, "ib-loopback/IB_write_32768_Avg_3:0": 22469.43, "ib-loopback/IB_write_65536_Avg_3:0": 22777.1, "ib-loopback/IB_write_131072_Avg_3:0": 23461.03, "ib-loopback/IB_write_262144_Avg_3:0": 23397.19, "ib-loopback/IB_write_524288_Avg_3:0": 23526.55, "ib-loopback/IB_write_1048576_Avg_3:0": 23854.76, "ib-loopback/IB_write_2097152_Avg_3:0": 23862.23, "ib-loopback/IB_write_4194304_Avg_3:0": 23931.15, "ib-loopback/IB_write_8388608_Avg_3:0": 23924.44, "ib-loopback/IB_write_512_Avg_4:0": 1523.33, "ib-loopback/IB_write_1024_Avg_4:0": 3233.23, "ib-loopback/IB_write_2048_Avg_4:0": 6792.88, "ib-loopback/IB_write_4096_Avg_4:0": 12616.05, "ib-loopback/IB_write_8192_Avg_4:0": 19324.05, "ib-loopback/IB_write_16384_Avg_4:0": 22082.51, "ib-loopback/IB_write_32768_Avg_4:0": 23294.23, "ib-loopback/IB_write_65536_Avg_4:0": 23546.22, "ib-loopback/IB_write_131072_Avg_4:0": 23727.91, "ib-loopback/IB_write_262144_Avg_4:0": 23843.93, "ib-loopback/IB_write_524288_Avg_4:0": 23905.96, "ib-loopback/IB_write_1048576_Avg_4:0": 23902.37, "ib-loopback/IB_write_2097152_Avg_4:0": 23921.03, "ib-loopback/IB_write_4194304_Avg_4:0": 23921.06, "ib-loopback/IB_write_8388608_Avg_4:0": 23922.4, "ib-loopback/IB_write_512_Avg_5:0": 1506.63, "ib-loopback/IB_write_1024_Avg_5:0": 3261.71, "ib-loopback/IB_write_2048_Avg_5:0": 6752.38, "ib-loopback/IB_write_4096_Avg_5:0": 13592.41, "ib-loopback/IB_write_8192_Avg_5:0": 19989.31, "ib-loopback/IB_write_16384_Avg_5:0": 22560.57, "ib-loopback/IB_write_32768_Avg_5:0": 23389.29, "ib-loopback/IB_write_65536_Avg_5:0": 23503.32, "ib-loopback/IB_write_131072_Avg_5:0": 23741.51, "ib-loopback/IB_write_262144_Avg_5:0": 23866.43, "ib-loopback/IB_write_524288_Avg_5:0": 23898.95, "ib-loopback/IB_write_1048576_Avg_5:0": 23876.36, "ib-loopback/IB_write_2097152_Avg_5:0": 23919.63, "ib-loopback/IB_write_4194304_Avg_5:0": 23924.68, "ib-loopback/IB_write_8388608_Avg_5:0": 23930.37, "ib-loopback/IB_write_512_Avg_6:0": 1467.69, "ib-loopback/IB_write_1024_Avg_6:0": 3157.04, "ib-loopback/IB_write_2048_Avg_6:0": 6494.61, "ib-loopback/IB_write_4096_Avg_6:0": 12883.51, "ib-loopback/IB_write_8192_Avg_6:0": 19207.67, "ib-loopback/IB_write_16384_Avg_6:0": 22519.39, "ib-loopback/IB_write_32768_Avg_6:0": 23323.46, "ib-loopback/IB_write_65536_Avg_6:0": 23523.6, "ib-loopback/IB_write_131072_Avg_6:0": 23626.67, "ib-loopback/IB_write_262144_Avg_6:0": 23836.99, "ib-loopback/IB_write_524288_Avg_6:0": 23904.51, "ib-loopback/IB_write_1048576_Avg_6:0": 23919.07, "ib-loopback/IB_write_2097152_Avg_6:0": 23943.82, "ib-loopback/IB_write_4194304_Avg_6:0": 23936.77, "ib-loopback/IB_write_8388608_Avg_6:0": 23941.57, "ib-loopback/IB_write_512_Avg_7:0": 1505.63, "ib-loopback/IB_write_1024_Avg_7:0": 3259.93, "ib-loopback/IB_write_2048_Avg_7:0": 6738.6, "ib-loopback/IB_write_4096_Avg_7:0": 13352.06, "ib-loopback/IB_write_8192_Avg_7:0": 19941.35, "ib-loopback/IB_write_16384_Avg_7:0": 22566.09, "ib-loopback/IB_write_32768_Avg_7:0": 23244.77, "ib-loopback/IB_write_65536_Avg_7:0": 23377.67, "ib-loopback/IB_write_131072_Avg_7:0": 23736.17, "ib-loopback/IB_write_262144_Avg_7:0": 23829.25, "ib-loopback/IB_write_524288_Avg_7:0": 23879.6, "ib-loopback/IB_write_1048576_Avg_7:0": 23895.1, "ib-loopback/IB_write_2097152_Avg_7:0": 23930.64, "ib-loopback/IB_write_4194304_Avg_7:0": 23845.63, "ib-loopback/IB_write_8388608_Avg_7:0": 23896.94, "kernel-launch/return_code":0, "kernel-launch/event_overhead:0": 0.00596, "kernel-launch/event_overhead:1": 0.00595, "kernel-launch/event_overhead:2": 0.00557, "kernel-launch/event_overhead:3": 0.0055, "kernel-launch/event_overhead:4": 0.00592, "kernel-launch/event_overhead:5": 0.00589, "kernel-launch/event_overhead:6": 0.00572, "kernel-launch/event_overhead:7": 0.0059, "kernel-launch/wall_overhead:0": 0.01026, "kernel-launch/wall_overhead:1": 0.01026, "kernel-launch/wall_overhead:2": 0.01046, "kernel-launch/wall_overhead:3": 0.01049, "kernel-launch/wall_overhead:4": 0.01063, "kernel-launch/wall_overhead:5": 0.01006, "kernel-launch/wall_overhead:6": 0.01045, "kernel-launch/wall_overhead:7": 0.01071, "lstm_models/pytorch-lstm/steptime_train_float32": 48.07024518959224, "lstm_models/pytorch-lstm/throughput_train_float32": 4806.472441132788, "lstm_models/pytorch-lstm/steptime_train_float16": 25.95312986522913, "lstm_models/pytorch-lstm/throughput_train_float16": 9069.90809255883, "pytorch-matmul/nosharding": 34.64499759674072, "mem-bw/return_code":1, "nccl-bw/allreduce_8_busbw:0": 0.0, "nccl-bw/allreduce_8_algbw:0": 0.0, "nccl-bw/allreduce_8_time:0": 37.84, "nccl-bw/allreduce_16_busbw:0": 0.0, "nccl-bw/allreduce_16_algbw:0": 0.0, "nccl-bw/allreduce_16_time:0": 36.42, "nccl-bw/allreduce_32_busbw:0": 0.0, "nccl-bw/allreduce_32_algbw:0": 0.0, "nccl-bw/allreduce_32_time:0": 36.87, "nccl-bw/allreduce_64_busbw:0": 0.0, "nccl-bw/allreduce_64_algbw:0": 0.0, "nccl-bw/allreduce_64_time:0": 35.83, "nccl-bw/allreduce_128_busbw:0": 0.01, "nccl-bw/allreduce_128_algbw:0": 0.0, "nccl-bw/allreduce_128_time:0": 36.91, "nccl-bw/allreduce_256_busbw:0": 0.01, "nccl-bw/allreduce_256_algbw:0": 0.01, "nccl-bw/allreduce_256_time:0": 37.58, "nccl-bw/allreduce_512_busbw:0": 0.02, "nccl-bw/allreduce_512_algbw:0": 0.01, "nccl-bw/allreduce_512_time:0": 36.98, "nccl-bw/allreduce_1024_busbw:0": 0.05, "nccl-bw/allreduce_1024_algbw:0": 0.03, "nccl-bw/allreduce_1024_time:0": 36.93, "nccl-bw/allreduce_2048_busbw:0": 0.1, "nccl-bw/allreduce_2048_algbw:0": 0.06, "nccl-bw/allreduce_2048_time:0": 36.06, "nccl-bw/allreduce_4096_busbw:0": 0.19, "nccl-bw/allreduce_4096_algbw:0": 0.11, "nccl-bw/allreduce_4096_time:0": 37.2, "nccl-bw/allreduce_8192_busbw:0": 0.39, "nccl-bw/allreduce_8192_algbw:0": 0.22, "nccl-bw/allreduce_8192_time:0": 37.04, "nccl-bw/allreduce_16384_busbw:0": 0.77, "nccl-bw/allreduce_16384_algbw:0": 0.44, "nccl-bw/allreduce_16384_time:0": 37.46, "nccl-bw/allreduce_32768_busbw:0": 1.52, "nccl-bw/allreduce_32768_algbw:0": 0.87, "nccl-bw/allreduce_32768_time:0": 37.64, "nccl-bw/allreduce_65536_busbw:0": 3.0, "nccl-bw/allreduce_65536_algbw:0": 1.71, "nccl-bw/allreduce_65536_time:0": 38.22, "nccl-bw/allreduce_131072_busbw:0": 5.31, "nccl-bw/allreduce_131072_algbw:0": 3.04, "nccl-bw/allreduce_131072_time:0": 43.17, "nccl-bw/allreduce_262144_busbw:0": 9.5, "nccl-bw/allreduce_262144_algbw:0": 5.43, "nccl-bw/allreduce_262144_time:0": 48.29, "nccl-bw/allreduce_524288_busbw:0": 15.11, "nccl-bw/allreduce_524288_algbw:0": 8.64, "nccl-bw/allreduce_524288_time:0": 60.71, "nccl-bw/allreduce_1048576_busbw:0": 24.1, "nccl-bw/allreduce_1048576_algbw:0": 13.77, "nccl-bw/allreduce_1048576_time:0": 76.13, "nccl-bw/allreduce_2097152_busbw:0": 38.12, "nccl-bw/allreduce_2097152_algbw:0": 21.78, "nccl-bw/allreduce_2097152_time:0": 96.28, "nccl-bw/allreduce_4194304_busbw:0": 65.75, "nccl-bw/allreduce_4194304_algbw:0": 37.57, "nccl-bw/allreduce_4194304_time:0": 111.6, "nccl-bw/allreduce_8388608_busbw:0": 89.51, "nccl-bw/allreduce_8388608_algbw:0": 51.15, "nccl-bw/allreduce_8388608_time:0": 164.0, "nccl-bw/allreduce_16777216_busbw:0": 114.38, "nccl-bw/allreduce_16777216_algbw:0": 65.36, "nccl-bw/allreduce_16777216_time:0": 256.7, "nccl-bw/allreduce_33554432_busbw:0": 154.89, "nccl-bw/allreduce_33554432_algbw:0": 88.51, "nccl-bw/allreduce_33554432_time:0": 379.1, "nccl-bw/allreduce_67108864_busbw:0": 200.01, "nccl-bw/allreduce_67108864_algbw:0": 114.29, "nccl-bw/allreduce_67108864_time:0": 587.2, "nccl-bw/allreduce_134217728_busbw:0": 202.97, "nccl-bw/allreduce_134217728_algbw:0": 115.98, "nccl-bw/allreduce_134217728_time:0": 1157.2, "nccl-bw/allreduce_268435456_busbw:0": 221.82, "nccl-bw/allreduce_268435456_algbw:0": 126.75, "nccl-bw/allreduce_268435456_time:0": 2117.8, "nccl-bw/allreduce_536870912_busbw:0": 224.54, "nccl-bw/allreduce_536870912_algbw:0": 128.31, "nccl-bw/allreduce_536870912_time:0": 4184.2, "nccl-bw/allreduce_1073741824_busbw:0": 230.15, "nccl-bw/allreduce_1073741824_algbw:0": 131.51, "nccl-bw/allreduce_1073741824_time:0": 8164.5, "nccl-bw/allreduce_2147483648_busbw:0": 231.89, "nccl-bw/allreduce_2147483648_algbw:0": 132.51, "nccl-bw/allreduce_2147483648_time:0": 16207.0, "nccl-bw/allreduce_4294967296_busbw:0": 234.45, "nccl-bw/allreduce_4294967296_algbw:0": 133.97, "nccl-bw/allreduce_4294967296_time:0": 32059.0, "nccl-bw/allreduce_8589934592_busbw:0": 235.36, "nccl-bw/allreduce_8589934592_algbw:0": 134.49, "nccl-bw/allreduce_8589934592_time:0": 63870.0, "resnet_models/pytorch-resnet50/steptime_train_float32": 253.95522732287645, "resnet_models/pytorch-resnet50/throughput_train_float32": 760.3348099129964, "resnet_models/pytorch-resnet50/steptime_train_float16": 200.08606184273958, "resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430922575, "resnet_models/pytorch-resnet101/steptime_train_float32": 389.08605091273785, "resnet_models/pytorch-resnet101/throughput_train_float32": 496.11747409298965, "resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918143, "resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195069, "resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278314769, "resnet_models/pytorch-resnet152/throughput_train_float32": 352.07099543348215, "resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802225, "resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998153649, "pytorch-sharding-matmul/allreduce": 10.574411869049072, "pytorch-sharding-matmul/allgather": 10.084696769714355, "vgg_models/pytorch-vgg11/steptime_train_float32": 40.35283671692014, "vgg_models/pytorch-vgg11/throughput_train_float32": 796.3615936949874, "vgg_models/pytorch-vgg11/steptime_train_float16": 24.133514845743775, "vgg_models/pytorch-vgg11/throughput_train_float16": 1330.411361458461, "vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936026275, "vgg_models/pytorch-vgg13/throughput_train_float32": 580.234107444399, "vgg_models/pytorch-vgg13/steptime_train_float16": 33.35228993091732, "vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023901524, "vgg_models/pytorch-vgg16/steptime_train_float32": 65.22519944701344, "vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638875934, "vgg_models/pytorch-vgg16/steptime_train_float16": 39.25287735182792, "vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546147621, "vgg_models/pytorch-vgg19/steptime_train_float32": 74.93487105239183, "vgg_models/pytorch-vgg19/throughput_train_float32": 429.80921583106164, "vgg_models/pytorch-vgg19/steptime_train_float16": 45.20330624654889, "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377091}
