diagnosis_summary.json 72.8 KB
Newer Older
1
2
[
    {
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
        "bert_models/pytorch-bert-base/steptime_train_float32": 114.5916701062,
        "bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591,
        "bert_models/pytorch-bert-base/steptime_train_float16": 83.8895108318,
        "bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582742,
        "bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371914,
        "bert_models/pytorch-bert-large/throughput_train_float32": 103.9487609742,
        "bert_models/pytorch-bert-large/steptime_train_float16": 206.8114168942,
        "bert_models/pytorch-bert-large/throughput_train_float16": 154.8408911711,
        "pytorch-computation-communication-overlap/mul_cost:0": 44.1822062144,
        "pytorch-computation-communication-overlap/mul_cost:1": 44.1822139389,
        "pytorch-computation-communication-overlap/mul_cost:2": 43.9701470781,
        "pytorch-computation-communication-overlap/mul_cost:3": 43.9701478756,
        "pytorch-computation-communication-overlap/mul_cost:4": 43.9701779317,
        "pytorch-computation-communication-overlap/mul_cost:5": 43.9701571606,
        "pytorch-computation-communication-overlap/mul_cost:6": 43.9701651983,
        "pytorch-computation-communication-overlap/mul_cost:7": 44.1795444785,
        "pytorch-computation-communication-overlap/matmul_cost:0": 137.0477370556,
        "pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344693,
        "pytorch-computation-communication-overlap/matmul_cost:2": 137.047772209,
        "pytorch-computation-communication-overlap/matmul_cost:3": 137.0477969726,
        "pytorch-computation-communication-overlap/matmul_cost:4": 137.0481367431,
        "pytorch-computation-communication-overlap/matmul_cost:5": 137.0482198877,
        "pytorch-computation-communication-overlap/matmul_cost:6": 137.0477532237,
        "pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081607,
        "densenet_models/pytorch-densenet169/steptime_train_float32": 150.6415554322,
        "densenet_models/pytorch-densenet169/throughput_train_float32": 212.4701719243,
        "densenet_models/pytorch-densenet169/steptime_train_float16": 145.3081957297,
        "densenet_models/pytorch-densenet169/throughput_train_float16": 220.2932337305,
        "densenet_models/pytorch-densenet201/steptime_train_float32": 182.9111778643,
        "densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232,
        "densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596,
        "densenet_models/pytorch-densenet201/throughput_train_float16": 181.5446573603,
        "gemm-flops/FP64:0": 9031.23,
        "gemm-flops/FP64:1": 9040.85,
        "gemm-flops/FP64:2": 9010.56,
        "gemm-flops/FP64:3": 9041.26,
        "gemm-flops/FP64:4": 9039.19,
        "gemm-flops/FP64:5": 9015.69,
        "gemm-flops/FP64:6": 9022.19,
        "gemm-flops/FP64:7": 9030.2,
        "gemm-flops/FP32:0": 18362.1,
        "gemm-flops/FP32:1": 18375.6,
        "gemm-flops/FP32:2": 18314.9,
        "gemm-flops/FP32:3": 18375.6,
        "gemm-flops/FP32:4": 18368.6,
        "gemm-flops/FP32:5": 18347.1,
        "gemm-flops/FP32:6": 18247.4,
        "gemm-flops/FP32:7": 18318.4,
        "gemm-flops/FP16:0": 33878.0,
        "gemm-flops/FP16:1": 33911.1,
        "gemm-flops/FP16:2": 33769.3,
        "gemm-flops/FP16:3": 33909.9,
        "gemm-flops/FP16:4": 33896.5,
        "gemm-flops/FP16:5": 33798.1,
        "gemm-flops/FP16:6": 33647.3,
        "gemm-flops/FP16:7": 33764.8,
        "gemm-flops/FP64_TC:0": 18963.6,
        "gemm-flops/FP64_TC:1": 18924.2,
        "gemm-flops/FP64_TC:2": 18930.3,
        "gemm-flops/FP64_TC:3": 18971.9,
        "gemm-flops/FP64_TC:4": 18946.0,
        "gemm-flops/FP64_TC:5": 18945.0,
        "gemm-flops/FP64_TC:6": 18822.9,
        "gemm-flops/FP64_TC:7": 18911.1,
        "gemm-flops/TF32_TC:0": 127900.0,
        "gemm-flops/TF32_TC:1": 129094.0,
        "gemm-flops/TF32_TC:2": 127831.0,
        "gemm-flops/TF32_TC:3": 128709.0,
        "gemm-flops/TF32_TC:4": 127388.0,
        "gemm-flops/TF32_TC:5": 127861.0,
        "gemm-flops/TF32_TC:6": 128492.0,
        "gemm-flops/TF32_TC:7": 127720.0,
        "gemm-flops/BF16_TC:0": 264965.0,
        "gemm-flops/BF16_TC:1": 266638.0,
        "gemm-flops/BF16_TC:2": 263151.0,
        "gemm-flops/BF16_TC:3": 264752.0,
        "gemm-flops/BF16_TC:4": 263049.0,
        "gemm-flops/BF16_TC:5": 266605.0,
        "gemm-flops/BF16_TC:6": 267501.0,
        "gemm-flops/BF16_TC:7": 263880.0,
        "gemm-flops/FP16_TC:0": 279474.0,
        "gemm-flops/FP16_TC:1": 281256.0,
        "gemm-flops/FP16_TC:2": 277403.0,
        "gemm-flops/FP16_TC:3": 279147.0,
        "gemm-flops/FP16_TC:4": 277587.0,
        "gemm-flops/FP16_TC:5": 281537.0,
        "gemm-flops/FP16_TC:6": 282132.0,
        "gemm-flops/FP16_TC:7": 277788.0,
        "gemm-flops/INT8_TC:0": 475160.0,
        "gemm-flops/INT8_TC:1": 477725.0,
        "gemm-flops/INT8_TC:2": 471621.0,
        "gemm-flops/INT8_TC:3": 473716.0,
        "gemm-flops/INT8_TC:4": 472124.0,
        "gemm-flops/INT8_TC:5": 479972.0,
        "gemm-flops/INT8_TC:6": 481327.0,
        "gemm-flops/INT8_TC:7": 474710.0,
        "gemm-flops/INT4_TC:0": 970330.0,
        "gemm-flops/INT4_TC:1": 976837.0,
        "gemm-flops/INT4_TC:2": 966003.0,
        "gemm-flops/INT4_TC:3": 971315.0,
        "gemm-flops/INT4_TC:4": 964441.0,
        "gemm-flops/INT4_TC:5": 982461.0,
        "gemm-flops/INT4_TC:6": 979610.0,
        "gemm-flops/INT4_TC:7": 968359.0,
        "gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971836,
        "gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.1154543969,
        "gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742235,
        "gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.1394499411,
        "gpu-sm-copy-bw/dtoh:0": 3.91755,
        "gpu-sm-copy-bw/dtoh:1": 4.45414,
        "gpu-sm-copy-bw/dtoh:2": 1.26483,
        "gpu-sm-copy-bw/dtoh:3": 1.30041,
        "gpu-sm-copy-bw/dtoh:4": 1.31577,
        "gpu-sm-copy-bw/dtoh:5": 1.27968,
        "gpu-sm-copy-bw/dtoh:6": 4.47849,
        "gpu-sm-copy-bw/dtoh:7": 3.96231,
        "gpu-sm-copy-bw/dtoh:8": 3.91705,
        "gpu-sm-copy-bw/dtoh:9": 4.45487,
        "gpu-sm-copy-bw/dtoh:10": 1.26352,
        "gpu-sm-copy-bw/dtoh:11": 1.2999,
        "gpu-sm-copy-bw/dtoh:12": 1.31677,
        "gpu-sm-copy-bw/dtoh:13": 1.27885,
        "gpu-sm-copy-bw/dtoh:14": 4.47913,
        "gpu-sm-copy-bw/dtoh:15": 3.95893,
        "gpu-sm-copy-bw/dtoh:16": 3.91729,
        "gpu-sm-copy-bw/dtoh:17": 4.45627,
        "gpu-sm-copy-bw/dtoh:18": 1.26437,
        "gpu-sm-copy-bw/dtoh:19": 1.30144,
        "gpu-sm-copy-bw/dtoh:20": 1.31704,
        "gpu-sm-copy-bw/dtoh:21": 1.27857,
        "gpu-sm-copy-bw/dtoh:22": 4.47889,
        "gpu-sm-copy-bw/dtoh:23": 3.95984,
        "gpu-sm-copy-bw/dtoh:24": 3.92025,
        "gpu-sm-copy-bw/dtoh:25": 4.45423,
        "gpu-sm-copy-bw/dtoh:26": 1.26449,
        "gpu-sm-copy-bw/dtoh:27": 1.29954,
        "gpu-sm-copy-bw/dtoh:28": 1.31731,
        "gpu-sm-copy-bw/dtoh:29": 1.27916,
        "gpu-sm-copy-bw/dtoh:30": 4.4797,
        "gpu-sm-copy-bw/dtoh:31": 3.96124,
        "gpu-sm-copy-bw/htod:0": 23.9685,
        "gpu-sm-copy-bw/htod:1": 23.967,
        "gpu-sm-copy-bw/htod:2": 19.9898,
        "gpu-sm-copy-bw/htod:3": 20.2848,
        "gpu-sm-copy-bw/htod:4": 20.3931,
        "gpu-sm-copy-bw/htod:5": 20.0888,
        "gpu-sm-copy-bw/htod:6": 23.9766,
        "gpu-sm-copy-bw/htod:7": 23.9792,
        "gpu-sm-copy-bw/htod:8": 23.9497,
        "gpu-sm-copy-bw/htod:9": 23.9438,
        "gpu-sm-copy-bw/htod:10": 20.0039,
        "gpu-sm-copy-bw/htod:11": 20.2469,
        "gpu-sm-copy-bw/htod:12": 20.4344,
        "gpu-sm-copy-bw/htod:13": 20.1005,
        "gpu-sm-copy-bw/htod:14": 23.9491,
        "gpu-sm-copy-bw/htod:15": 23.9898,
        "gpu-sm-copy-bw/htod:16": 23.962,
        "gpu-sm-copy-bw/htod:17": 23.97,
        "gpu-sm-copy-bw/htod:18": 19.987,
        "gpu-sm-copy-bw/htod:19": 20.1549,
        "gpu-sm-copy-bw/htod:20": 20.3931,
        "gpu-sm-copy-bw/htod:21": 20.1013,
        "gpu-sm-copy-bw/htod:22": 23.9865,
        "gpu-sm-copy-bw/htod:23": 23.972,
        "gpu-sm-copy-bw/htod:24": 23.9569,
        "gpu-sm-copy-bw/htod:25": 23.9762,
        "gpu-sm-copy-bw/htod:26": 19.92,
        "gpu-sm-copy-bw/htod:27": 20.2638,
        "gpu-sm-copy-bw/htod:28": 20.4419,
        "gpu-sm-copy-bw/htod:29": 20.1054,
        "gpu-sm-copy-bw/htod:30": 23.9752,
        "gpu-sm-copy-bw/htod:31": 23.9631,
        "ib-loopback/IB_write_512_Avg_0:0": 1492.89,
        "ib-loopback/IB_write_1024_Avg_0:0": 3224.92,
        "ib-loopback/IB_write_2048_Avg_0:0": 6714.5,
        "ib-loopback/IB_write_4096_Avg_0:0": 12871.93,
        "ib-loopback/IB_write_8192_Avg_0:0": 19990.78,
        "ib-loopback/IB_write_16384_Avg_0:0": 22172.25,
        "ib-loopback/IB_write_32768_Avg_0:0": 23073.19,
        "ib-loopback/IB_write_65536_Avg_0:0": 23527.09,
        "ib-loopback/IB_write_131072_Avg_0:0": 23805.92,
        "ib-loopback/IB_write_262144_Avg_0:0": 23380.51,
        "ib-loopback/IB_write_524288_Avg_0:0": 23856.63,
        "ib-loopback/IB_write_1048576_Avg_0:0": 23869.58,
        "ib-loopback/IB_write_2097152_Avg_0:0": 23885.7,
        "ib-loopback/IB_write_4194304_Avg_0:0": 23914.56,
        "ib-loopback/IB_write_8388608_Avg_0:0": 23935.21,
        "ib-loopback/IB_write_512_Avg_1:0": 1479.1,
        "ib-loopback/IB_write_1024_Avg_1:0": 3246.48,
        "ib-loopback/IB_write_2048_Avg_1:0": 6754.94,
        "ib-loopback/IB_write_4096_Avg_1:0": 13101.09,
        "ib-loopback/IB_write_8192_Avg_1:0": 19945.81,
        "ib-loopback/IB_write_16384_Avg_1:0": 22631.62,
        "ib-loopback/IB_write_32768_Avg_1:0": 23381.58,
        "ib-loopback/IB_write_65536_Avg_1:0": 23515.29,
        "ib-loopback/IB_write_131072_Avg_1:0": 23756.01,
        "ib-loopback/IB_write_262144_Avg_1:0": 23875.91,
        "ib-loopback/IB_write_524288_Avg_1:0": 23911.13,
        "ib-loopback/IB_write_1048576_Avg_1:0": 23935.42,
        "ib-loopback/IB_write_2097152_Avg_1:0": 23941.25,
        "ib-loopback/IB_write_4194304_Avg_1:0": 23922.06,
        "ib-loopback/IB_write_8388608_Avg_1:0": 23928.59,
        "ib-loopback/IB_write_512_Avg_2:0": 1505.29,
        "ib-loopback/IB_write_1024_Avg_2:0": 3215.97,
        "ib-loopback/IB_write_2048_Avg_2:0": 6745.49,
        "ib-loopback/IB_write_4096_Avg_2:0": 11548.88,
        "ib-loopback/IB_write_8192_Avg_2:0": 19432.15,
        "ib-loopback/IB_write_16384_Avg_2:0": 22765.51,
        "ib-loopback/IB_write_32768_Avg_2:0": 23235.07,
        "ib-loopback/IB_write_65536_Avg_2:0": 23620.08,
        "ib-loopback/IB_write_131072_Avg_2:0": 23759.08,
        "ib-loopback/IB_write_262144_Avg_2:0": 23859.82,
        "ib-loopback/IB_write_524288_Avg_2:0": 23775.01,
        "ib-loopback/IB_write_1048576_Avg_2:0": 23885.96,
        "ib-loopback/IB_write_2097152_Avg_2:0": 23894.73,
        "ib-loopback/IB_write_4194304_Avg_2:0": 23909.98,
        "ib-loopback/IB_write_8388608_Avg_2:0": 23927.21,
        "ib-loopback/IB_write_512_Avg_3:0": 1474.47,
        "ib-loopback/IB_write_1024_Avg_3:0": 3344.68,
        "ib-loopback/IB_write_2048_Avg_3:0": 6606.35,
        "ib-loopback/IB_write_4096_Avg_3:0": 12071.4,
        "ib-loopback/IB_write_8192_Avg_3:0": 18408.33,
        "ib-loopback/IB_write_16384_Avg_3:0": 20789.27,
        "ib-loopback/IB_write_32768_Avg_3:0": 22469.43,
        "ib-loopback/IB_write_65536_Avg_3:0": 22777.1,
        "ib-loopback/IB_write_131072_Avg_3:0": 23461.03,
        "ib-loopback/IB_write_262144_Avg_3:0": 23397.19,
        "ib-loopback/IB_write_524288_Avg_3:0": 23526.55,
        "ib-loopback/IB_write_1048576_Avg_3:0": 23854.76,
        "ib-loopback/IB_write_2097152_Avg_3:0": 23862.23,
        "ib-loopback/IB_write_4194304_Avg_3:0": 23931.15,
        "ib-loopback/IB_write_8388608_Avg_3:0": 23924.44,
        "ib-loopback/IB_write_512_Avg_4:0": 1523.33,
        "ib-loopback/IB_write_1024_Avg_4:0": 3233.23,
        "ib-loopback/IB_write_2048_Avg_4:0": 6792.88,
        "ib-loopback/IB_write_4096_Avg_4:0": 12616.05,
        "ib-loopback/IB_write_8192_Avg_4:0": 19324.05,
        "ib-loopback/IB_write_16384_Avg_4:0": 22082.51,
        "ib-loopback/IB_write_32768_Avg_4:0": 23294.23,
        "ib-loopback/IB_write_65536_Avg_4:0": 23546.22,
        "ib-loopback/IB_write_131072_Avg_4:0": 23727.91,
        "ib-loopback/IB_write_262144_Avg_4:0": 23843.93,
        "ib-loopback/IB_write_524288_Avg_4:0": 23905.96,
        "ib-loopback/IB_write_1048576_Avg_4:0": 23902.37,
        "ib-loopback/IB_write_2097152_Avg_4:0": 23921.03,
        "ib-loopback/IB_write_4194304_Avg_4:0": 23921.06,
        "ib-loopback/IB_write_8388608_Avg_4:0": 23922.4,
        "ib-loopback/IB_write_512_Avg_5:0": 1506.63,
        "ib-loopback/IB_write_1024_Avg_5:0": 3261.71,
        "ib-loopback/IB_write_2048_Avg_5:0": 6752.38,
        "ib-loopback/IB_write_4096_Avg_5:0": 13592.41,
        "ib-loopback/IB_write_8192_Avg_5:0": 19989.31,
        "ib-loopback/IB_write_16384_Avg_5:0": 22560.57,
        "ib-loopback/IB_write_32768_Avg_5:0": 23389.29,
        "ib-loopback/IB_write_65536_Avg_5:0": 23503.32,
        "ib-loopback/IB_write_131072_Avg_5:0": 23741.51,
        "ib-loopback/IB_write_262144_Avg_5:0": 23866.43,
        "ib-loopback/IB_write_524288_Avg_5:0": 23898.95,
        "ib-loopback/IB_write_1048576_Avg_5:0": 23876.36,
        "ib-loopback/IB_write_2097152_Avg_5:0": 23919.63,
        "ib-loopback/IB_write_4194304_Avg_5:0": 23924.68,
        "ib-loopback/IB_write_8388608_Avg_5:0": 23930.37,
        "ib-loopback/IB_write_512_Avg_6:0": 1467.69,
        "ib-loopback/IB_write_1024_Avg_6:0": 3157.04,
        "ib-loopback/IB_write_2048_Avg_6:0": 6494.61,
        "ib-loopback/IB_write_4096_Avg_6:0": 12883.51,
        "ib-loopback/IB_write_8192_Avg_6:0": 19207.67,
        "ib-loopback/IB_write_16384_Avg_6:0": 22519.39,
        "ib-loopback/IB_write_32768_Avg_6:0": 23323.46,
        "ib-loopback/IB_write_65536_Avg_6:0": 23523.6,
        "ib-loopback/IB_write_131072_Avg_6:0": 23626.67,
        "ib-loopback/IB_write_262144_Avg_6:0": 23836.99,
        "ib-loopback/IB_write_524288_Avg_6:0": 23904.51,
        "ib-loopback/IB_write_1048576_Avg_6:0": 23919.07,
        "ib-loopback/IB_write_2097152_Avg_6:0": 23943.82,
        "ib-loopback/IB_write_4194304_Avg_6:0": 23936.77,
        "ib-loopback/IB_write_8388608_Avg_6:0": 23941.57,
        "ib-loopback/IB_write_512_Avg_7:0": 1505.63,
        "ib-loopback/IB_write_1024_Avg_7:0": 3259.93,
        "ib-loopback/IB_write_2048_Avg_7:0": 6738.6,
        "ib-loopback/IB_write_4096_Avg_7:0": 13352.06,
        "ib-loopback/IB_write_8192_Avg_7:0": 19941.35,
        "ib-loopback/IB_write_16384_Avg_7:0": 22566.09,
        "ib-loopback/IB_write_32768_Avg_7:0": 23244.77,
        "ib-loopback/IB_write_65536_Avg_7:0": 23377.67,
        "ib-loopback/IB_write_131072_Avg_7:0": 23736.17,
        "ib-loopback/IB_write_262144_Avg_7:0": 23829.25,
        "ib-loopback/IB_write_524288_Avg_7:0": 23879.6,
        "ib-loopback/IB_write_1048576_Avg_7:0": 23895.1,
        "ib-loopback/IB_write_2097152_Avg_7:0": 23930.64,
        "ib-loopback/IB_write_4194304_Avg_7:0": 23845.63,
        "ib-loopback/IB_write_8388608_Avg_7:0": 23896.94,
        "kernel-launch/return_code": 0.0,
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
        "kernel-launch/event_overhead:0": 0.1,
        "kernel-launch/event_overhead:1": 0.00595,
        "kernel-launch/event_overhead:2": 0.00557,
        "kernel-launch/event_overhead:3": 0.0055,
        "kernel-launch/event_overhead:4": 0.00592,
        "kernel-launch/event_overhead:5": 0.00589,
        "kernel-launch/event_overhead:6": 0.00572,
        "kernel-launch/event_overhead:7": 0.0059,
        "kernel-launch/wall_overhead:0": 0.01026,
        "kernel-launch/wall_overhead:1": 0.01026,
        "kernel-launch/wall_overhead:2": 0.01046,
        "kernel-launch/wall_overhead:3": 0.01049,
        "kernel-launch/wall_overhead:4": 0.01063,
        "kernel-launch/wall_overhead:5": 0.01006,
        "kernel-launch/wall_overhead:6": 0.01045,
        "kernel-launch/wall_overhead:7": 0.01071,
312
313
314
315
316
317
        "lstm_models/pytorch-lstm/steptime_train_float32": 48.0702451896,
        "lstm_models/pytorch-lstm/throughput_train_float32": 4806.4724411328,
        "lstm_models/pytorch-lstm/steptime_train_float16": 25.9531298652,
        "lstm_models/pytorch-lstm/throughput_train_float16": 9069.9080925588,
        "pytorch-matmul/nosharding": 34.6449975967,
        "mem-bw/return_code": 0.0,
318
319
320
321
322
323
324
325
        "mem-bw/H2D_Mem_BW:0": 25.6,
        "mem-bw/H2D_Mem_BW:1": 25.8,
        "mem-bw/H2D_Mem_BW:2": 26.0,
        "mem-bw/H2D_Mem_BW:3": 26.1,
        "mem-bw/H2D_Mem_BW:4": 26.2,
        "mem-bw/H2D_Mem_BW:5": 25.8,
        "mem-bw/H2D_Mem_BW:6": 25.3,
        "mem-bw/H2D_Mem_BW:7": 26.1,
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
        "mem-bw/D2H_Mem_BW:0": 24.3,
        "mem-bw/D2H_Mem_BW:1": 24.6,
        "mem-bw/D2H_Mem_BW:2": 24.5,
        "mem-bw/D2H_Mem_BW:3": 24.6,
        "mem-bw/D2H_Mem_BW:4": 24.3,
        "mem-bw/D2H_Mem_BW:5": 24.3,
        "mem-bw/D2H_Mem_BW:6": 23.9,
        "mem-bw/D2H_Mem_BW:7": 24.6,
        "mem-bw/D2D_Mem_BW:0": 1118.0,
        "mem-bw/D2D_Mem_BW:1": 1114.6,
        "mem-bw/D2D_Mem_BW:2": 1119.7,
        "mem-bw/D2D_Mem_BW:3": 1121.9,
        "mem-bw/D2D_Mem_BW:4": 1109.7,
        "mem-bw/D2D_Mem_BW:5": 1110.1,
        "mem-bw/D2D_Mem_BW:6": 1123.3,
        "mem-bw/D2D_Mem_BW:7": 1117.6,
        "nccl-bw/allreduce_8_busbw:0": 0.0,
        "nccl-bw/allreduce_8_algbw:0": 0.0,
        "nccl-bw/allreduce_8_time:0": 37.84,
        "nccl-bw/allreduce_16_busbw:0": 0.0,
        "nccl-bw/allreduce_16_algbw:0": 0.0,
        "nccl-bw/allreduce_16_time:0": 36.42,
        "nccl-bw/allreduce_32_busbw:0": 0.0,
        "nccl-bw/allreduce_32_algbw:0": 0.0,
        "nccl-bw/allreduce_32_time:0": 36.87,
        "nccl-bw/allreduce_64_busbw:0": 0.0,
        "nccl-bw/allreduce_64_algbw:0": 0.0,
        "nccl-bw/allreduce_64_time:0": 35.83,
        "nccl-bw/allreduce_128_busbw:0": 0.01,
        "nccl-bw/allreduce_128_algbw:0": 0.0,
        "nccl-bw/allreduce_128_time:0": 36.91,
        "nccl-bw/allreduce_256_busbw:0": 0.01,
        "nccl-bw/allreduce_256_algbw:0": 0.01,
        "nccl-bw/allreduce_256_time:0": 37.58,
        "nccl-bw/allreduce_512_busbw:0": 0.02,
        "nccl-bw/allreduce_512_algbw:0": 0.01,
        "nccl-bw/allreduce_512_time:0": 36.98,
        "nccl-bw/allreduce_1024_busbw:0": 0.05,
        "nccl-bw/allreduce_1024_algbw:0": 0.03,
        "nccl-bw/allreduce_1024_time:0": 36.93,
        "nccl-bw/allreduce_2048_busbw:0": 0.1,
        "nccl-bw/allreduce_2048_algbw:0": 0.06,
        "nccl-bw/allreduce_2048_time:0": 36.06,
        "nccl-bw/allreduce_4096_busbw:0": 0.19,
        "nccl-bw/allreduce_4096_algbw:0": 0.11,
        "nccl-bw/allreduce_4096_time:0": 37.2,
        "nccl-bw/allreduce_8192_busbw:0": 0.39,
        "nccl-bw/allreduce_8192_algbw:0": 0.22,
        "nccl-bw/allreduce_8192_time:0": 37.04,
        "nccl-bw/allreduce_16384_busbw:0": 0.77,
        "nccl-bw/allreduce_16384_algbw:0": 0.44,
        "nccl-bw/allreduce_16384_time:0": 37.46,
        "nccl-bw/allreduce_32768_busbw:0": 1.52,
        "nccl-bw/allreduce_32768_algbw:0": 0.87,
        "nccl-bw/allreduce_32768_time:0": 37.64,
        "nccl-bw/allreduce_65536_busbw:0": 3.0,
        "nccl-bw/allreduce_65536_algbw:0": 1.71,
        "nccl-bw/allreduce_65536_time:0": 38.22,
        "nccl-bw/allreduce_131072_busbw:0": 5.31,
        "nccl-bw/allreduce_131072_algbw:0": 3.04,
        "nccl-bw/allreduce_131072_time:0": 43.17,
        "nccl-bw/allreduce_262144_busbw:0": 9.5,
        "nccl-bw/allreduce_262144_algbw:0": 5.43,
        "nccl-bw/allreduce_262144_time:0": 48.29,
        "nccl-bw/allreduce_524288_busbw:0": 15.11,
        "nccl-bw/allreduce_524288_algbw:0": 8.64,
        "nccl-bw/allreduce_524288_time:0": 60.71,
        "nccl-bw/allreduce_1048576_busbw:0": 24.1,
        "nccl-bw/allreduce_1048576_algbw:0": 13.77,
        "nccl-bw/allreduce_1048576_time:0": 76.13,
        "nccl-bw/allreduce_2097152_busbw:0": 38.12,
        "nccl-bw/allreduce_2097152_algbw:0": 21.78,
        "nccl-bw/allreduce_2097152_time:0": 96.28,
        "nccl-bw/allreduce_4194304_busbw:0": 65.75,
        "nccl-bw/allreduce_4194304_algbw:0": 37.57,
        "nccl-bw/allreduce_4194304_time:0": 111.6,
        "nccl-bw/allreduce_8388608_busbw:0": 89.51,
        "nccl-bw/allreduce_8388608_algbw:0": 51.15,
        "nccl-bw/allreduce_8388608_time:0": 164.0,
        "nccl-bw/allreduce_16777216_busbw:0": 114.38,
        "nccl-bw/allreduce_16777216_algbw:0": 65.36,
        "nccl-bw/allreduce_16777216_time:0": 256.7,
        "nccl-bw/allreduce_33554432_busbw:0": 154.89,
        "nccl-bw/allreduce_33554432_algbw:0": 88.51,
        "nccl-bw/allreduce_33554432_time:0": 379.1,
        "nccl-bw/allreduce_67108864_busbw:0": 200.01,
        "nccl-bw/allreduce_67108864_algbw:0": 114.29,
        "nccl-bw/allreduce_67108864_time:0": 587.2,
        "nccl-bw/allreduce_134217728_busbw:0": 202.97,
        "nccl-bw/allreduce_134217728_algbw:0": 115.98,
        "nccl-bw/allreduce_134217728_time:0": 1157.2,
        "nccl-bw/allreduce_268435456_busbw:0": 221.82,
        "nccl-bw/allreduce_268435456_algbw:0": 126.75,
        "nccl-bw/allreduce_268435456_time:0": 2117.8,
        "nccl-bw/allreduce_536870912_busbw:0": 224.54,
        "nccl-bw/allreduce_536870912_algbw:0": 128.31,
        "nccl-bw/allreduce_536870912_time:0": 4184.2,
        "nccl-bw/allreduce_1073741824_busbw:0": 230.15,
        "nccl-bw/allreduce_1073741824_algbw:0": 131.51,
        "nccl-bw/allreduce_1073741824_time:0": 8164.5,
        "nccl-bw/allreduce_2147483648_busbw:0": 231.89,
        "nccl-bw/allreduce_2147483648_algbw:0": 132.51,
        "nccl-bw/allreduce_2147483648_time:0": 16207.0,
        "nccl-bw/allreduce_4294967296_busbw:0": 234.45,
        "nccl-bw/allreduce_4294967296_algbw:0": 133.97,
        "nccl-bw/allreduce_4294967296_time:0": 32059.0,
        "nccl-bw/allreduce_8589934592_busbw:0": 235.36,
        "nccl-bw/allreduce_8589934592_algbw:0": 134.49,
        "nccl-bw/allreduce_8589934592_time:0": 63870.0,
        "resnet_models/pytorch-resnet50/steptime_train_float32": 253.9552273229,
        "resnet_models/pytorch-resnet50/throughput_train_float32": 760.334809913,
        "resnet_models/pytorch-resnet50/steptime_train_float16": 200.0860618427,
        "resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430923,
        "resnet_models/pytorch-resnet101/steptime_train_float32": 389.0860509127,
        "resnet_models/pytorch-resnet101/throughput_train_float32": 496.117474093,
        "resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918,
        "resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195,
        "resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278315,
        "resnet_models/pytorch-resnet152/throughput_train_float32": 352.0709954335,
        "resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802,
        "resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998154,
        "pytorch-sharding-matmul/allreduce": 10.574411869,
        "pytorch-sharding-matmul/allgather": 10.0846967697,
        "vgg_models/pytorch-vgg11/steptime_train_float32": 40.3528367169,
        "vgg_models/pytorch-vgg11/throughput_train_float32": 796.361593695,
        "vgg_models/pytorch-vgg11/steptime_train_float16": 24.1335148457,
        "vgg_models/pytorch-vgg11/throughput_train_float16": 1330.4113614585,
        "vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936,
        "vgg_models/pytorch-vgg13/throughput_train_float32": 580.2341074444,
        "vgg_models/pytorch-vgg13/steptime_train_float16": 33.3522899309,
        "vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023902,
        "vgg_models/pytorch-vgg16/steptime_train_float32": 65.225199447,
        "vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638876,
        "vgg_models/pytorch-vgg16/steptime_train_float16": 39.2528773518,
        "vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546148,
        "vgg_models/pytorch-vgg19/steptime_train_float32": 74.9348710524,
        "vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
        "vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
        "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377,
465
466
467
468
        "diagnosis/accept": false,
        "diagnosis/issue_num": 1,
        "diagnosis/category": "KernelLaunch",
        "diagnosis/issue_details": "kernel-launch/event_overhead:0(B/L: 0.0060 VAL: 0.1000 VAR: 1577.85% Rule:lambda x:x>0.05)",
469
470
471
        "Index": "sb-validation-01"
    },
    {
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
        "bert_models/pytorch-bert-base/steptime_train_float32": 114.5916701062,
        "bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591,
        "bert_models/pytorch-bert-base/steptime_train_float16": 83.8895108318,
        "bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582742,
        "bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371914,
        "bert_models/pytorch-bert-large/throughput_train_float32": 103.9487609742,
        "bert_models/pytorch-bert-large/steptime_train_float16": 206.8114168942,
        "bert_models/pytorch-bert-large/throughput_train_float16": 154.8408911711,
        "pytorch-computation-communication-overlap/mul_cost:0": 44.1822062144,
        "pytorch-computation-communication-overlap/mul_cost:1": 44.1822139389,
        "pytorch-computation-communication-overlap/mul_cost:2": 43.9701470781,
        "pytorch-computation-communication-overlap/mul_cost:3": 43.9701478756,
        "pytorch-computation-communication-overlap/mul_cost:4": 43.9701779317,
        "pytorch-computation-communication-overlap/mul_cost:5": 43.9701571606,
        "pytorch-computation-communication-overlap/mul_cost:6": 43.9701651983,
        "pytorch-computation-communication-overlap/mul_cost:7": 44.1795444785,
        "pytorch-computation-communication-overlap/matmul_cost:0": 137.0477370556,
        "pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344693,
        "pytorch-computation-communication-overlap/matmul_cost:2": 137.047772209,
        "pytorch-computation-communication-overlap/matmul_cost:3": 137.0477969726,
        "pytorch-computation-communication-overlap/matmul_cost:4": 137.0481367431,
        "pytorch-computation-communication-overlap/matmul_cost:5": 137.0482198877,
        "pytorch-computation-communication-overlap/matmul_cost:6": 137.0477532237,
        "pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081607,
        "densenet_models/pytorch-densenet169/steptime_train_float32": 150.6415554322,
        "densenet_models/pytorch-densenet169/throughput_train_float32": 212.4701719243,
        "densenet_models/pytorch-densenet169/steptime_train_float16": 145.3081957297,
        "densenet_models/pytorch-densenet169/throughput_train_float16": 220.2932337305,
        "densenet_models/pytorch-densenet201/steptime_train_float32": 182.9111778643,
        "densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232,
        "densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596,
        "densenet_models/pytorch-densenet201/throughput_train_float16": 181.5446573603,
        "gemm-flops/FP64:0": 9031.23,
        "gemm-flops/FP64:1": 9040.85,
        "gemm-flops/FP64:2": 9010.56,
        "gemm-flops/FP64:3": 9041.26,
        "gemm-flops/FP64:4": 9039.19,
        "gemm-flops/FP64:5": 9015.69,
        "gemm-flops/FP64:6": 9022.19,
        "gemm-flops/FP64:7": 9030.2,
        "gemm-flops/FP32:0": 18362.1,
        "gemm-flops/FP32:1": 18375.6,
        "gemm-flops/FP32:2": 18314.9,
        "gemm-flops/FP32:3": 18375.6,
        "gemm-flops/FP32:4": 18368.6,
        "gemm-flops/FP32:5": 18347.1,
        "gemm-flops/FP32:6": 18247.4,
        "gemm-flops/FP32:7": 18318.4,
        "gemm-flops/FP16:0": 33878.0,
        "gemm-flops/FP16:1": 33911.1,
        "gemm-flops/FP16:2": 33769.3,
        "gemm-flops/FP16:3": 33909.9,
        "gemm-flops/FP16:4": 33896.5,
        "gemm-flops/FP16:5": 33798.1,
        "gemm-flops/FP16:6": 33647.3,
        "gemm-flops/FP16:7": 33764.8,
        "gemm-flops/FP64_TC:0": 18963.6,
        "gemm-flops/FP64_TC:1": 18924.2,
        "gemm-flops/FP64_TC:2": 18930.3,
        "gemm-flops/FP64_TC:3": 18971.9,
        "gemm-flops/FP64_TC:4": 18946.0,
        "gemm-flops/FP64_TC:5": 18945.0,
        "gemm-flops/FP64_TC:6": 18822.9,
        "gemm-flops/FP64_TC:7": 18911.1,
        "gemm-flops/TF32_TC:0": 127900.0,
        "gemm-flops/TF32_TC:1": 129094.0,
        "gemm-flops/TF32_TC:2": 127831.0,
        "gemm-flops/TF32_TC:3": 128709.0,
        "gemm-flops/TF32_TC:4": 127388.0,
        "gemm-flops/TF32_TC:5": 127861.0,
        "gemm-flops/TF32_TC:6": 128492.0,
        "gemm-flops/TF32_TC:7": 127720.0,
        "gemm-flops/BF16_TC:0": 264965.0,
        "gemm-flops/BF16_TC:1": 266638.0,
        "gemm-flops/BF16_TC:2": 263151.0,
        "gemm-flops/BF16_TC:3": 264752.0,
        "gemm-flops/BF16_TC:4": 263049.0,
        "gemm-flops/BF16_TC:5": 266605.0,
        "gemm-flops/BF16_TC:6": 267501.0,
        "gemm-flops/BF16_TC:7": 263880.0,
        "gemm-flops/FP16_TC:0": 279474.0,
        "gemm-flops/FP16_TC:1": 281256.0,
        "gemm-flops/FP16_TC:2": 277403.0,
        "gemm-flops/FP16_TC:3": 279147.0,
        "gemm-flops/FP16_TC:4": 277587.0,
        "gemm-flops/FP16_TC:5": 281537.0,
        "gemm-flops/FP16_TC:6": 282132.0,
        "gemm-flops/FP16_TC:7": 277788.0,
        "gemm-flops/INT8_TC:0": 475160.0,
        "gemm-flops/INT8_TC:1": 477725.0,
        "gemm-flops/INT8_TC:2": 471621.0,
        "gemm-flops/INT8_TC:3": 473716.0,
        "gemm-flops/INT8_TC:4": 472124.0,
        "gemm-flops/INT8_TC:5": 479972.0,
        "gemm-flops/INT8_TC:6": 481327.0,
        "gemm-flops/INT8_TC:7": 474710.0,
        "gemm-flops/INT4_TC:0": 970330.0,
        "gemm-flops/INT4_TC:1": 976837.0,
        "gemm-flops/INT4_TC:2": 966003.0,
        "gemm-flops/INT4_TC:3": 971315.0,
        "gemm-flops/INT4_TC:4": 964441.0,
        "gemm-flops/INT4_TC:5": 982461.0,
        "gemm-flops/INT4_TC:6": 979610.0,
        "gemm-flops/INT4_TC:7": 968359.0,
        "gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971836,
        "gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.1154543969,
        "gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742235,
        "gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.1394499411,
        "gpu-sm-copy-bw/dtoh:0": 3.91755,
        "gpu-sm-copy-bw/dtoh:1": 4.45414,
        "gpu-sm-copy-bw/dtoh:2": 1.26483,
        "gpu-sm-copy-bw/dtoh:3": 1.30041,
        "gpu-sm-copy-bw/dtoh:4": 1.31577,
        "gpu-sm-copy-bw/dtoh:5": 1.27968,
        "gpu-sm-copy-bw/dtoh:6": 4.47849,
        "gpu-sm-copy-bw/dtoh:7": 3.96231,
        "gpu-sm-copy-bw/dtoh:8": 3.91705,
        "gpu-sm-copy-bw/dtoh:9": 4.45487,
        "gpu-sm-copy-bw/dtoh:10": 1.26352,
        "gpu-sm-copy-bw/dtoh:11": 1.2999,
        "gpu-sm-copy-bw/dtoh:12": 1.31677,
        "gpu-sm-copy-bw/dtoh:13": 1.27885,
        "gpu-sm-copy-bw/dtoh:14": 4.47913,
        "gpu-sm-copy-bw/dtoh:15": 3.95893,
        "gpu-sm-copy-bw/dtoh:16": 3.91729,
        "gpu-sm-copy-bw/dtoh:17": 4.45627,
        "gpu-sm-copy-bw/dtoh:18": 1.26437,
        "gpu-sm-copy-bw/dtoh:19": 1.30144,
        "gpu-sm-copy-bw/dtoh:20": 1.31704,
        "gpu-sm-copy-bw/dtoh:21": 1.27857,
        "gpu-sm-copy-bw/dtoh:22": 4.47889,
        "gpu-sm-copy-bw/dtoh:23": 3.95984,
        "gpu-sm-copy-bw/dtoh:24": 3.92025,
        "gpu-sm-copy-bw/dtoh:25": 4.45423,
        "gpu-sm-copy-bw/dtoh:26": 1.26449,
        "gpu-sm-copy-bw/dtoh:27": 1.29954,
        "gpu-sm-copy-bw/dtoh:28": 1.31731,
        "gpu-sm-copy-bw/dtoh:29": 1.27916,
        "gpu-sm-copy-bw/dtoh:30": 4.4797,
        "gpu-sm-copy-bw/dtoh:31": 3.96124,
        "gpu-sm-copy-bw/htod:0": 23.9685,
        "gpu-sm-copy-bw/htod:1": 23.967,
        "gpu-sm-copy-bw/htod:2": 19.9898,
        "gpu-sm-copy-bw/htod:3": 20.2848,
        "gpu-sm-copy-bw/htod:4": 20.3931,
        "gpu-sm-copy-bw/htod:5": 20.0888,
        "gpu-sm-copy-bw/htod:6": 23.9766,
        "gpu-sm-copy-bw/htod:7": 23.9792,
        "gpu-sm-copy-bw/htod:8": 23.9497,
        "gpu-sm-copy-bw/htod:9": 23.9438,
        "gpu-sm-copy-bw/htod:10": 20.0039,
        "gpu-sm-copy-bw/htod:11": 20.2469,
        "gpu-sm-copy-bw/htod:12": 20.4344,
        "gpu-sm-copy-bw/htod:13": 20.1005,
        "gpu-sm-copy-bw/htod:14": 23.9491,
        "gpu-sm-copy-bw/htod:15": 23.9898,
        "gpu-sm-copy-bw/htod:16": 23.962,
        "gpu-sm-copy-bw/htod:17": 23.97,
        "gpu-sm-copy-bw/htod:18": 19.987,
        "gpu-sm-copy-bw/htod:19": 20.1549,
        "gpu-sm-copy-bw/htod:20": 20.3931,
        "gpu-sm-copy-bw/htod:21": 20.1013,
        "gpu-sm-copy-bw/htod:22": 23.9865,
        "gpu-sm-copy-bw/htod:23": 23.972,
        "gpu-sm-copy-bw/htod:24": 23.9569,
        "gpu-sm-copy-bw/htod:25": 23.9762,
        "gpu-sm-copy-bw/htod:26": 19.92,
        "gpu-sm-copy-bw/htod:27": 20.2638,
        "gpu-sm-copy-bw/htod:28": 20.4419,
        "gpu-sm-copy-bw/htod:29": 20.1054,
        "gpu-sm-copy-bw/htod:30": 23.9752,
        "gpu-sm-copy-bw/htod:31": 23.9631,
        "ib-loopback/IB_write_512_Avg_0:0": 1492.89,
        "ib-loopback/IB_write_1024_Avg_0:0": 3224.92,
        "ib-loopback/IB_write_2048_Avg_0:0": 6714.5,
        "ib-loopback/IB_write_4096_Avg_0:0": 12871.93,
        "ib-loopback/IB_write_8192_Avg_0:0": 19990.78,
        "ib-loopback/IB_write_16384_Avg_0:0": 22172.25,
        "ib-loopback/IB_write_32768_Avg_0:0": 23073.19,
        "ib-loopback/IB_write_65536_Avg_0:0": 23527.09,
        "ib-loopback/IB_write_131072_Avg_0:0": 23805.92,
        "ib-loopback/IB_write_262144_Avg_0:0": 23380.51,
        "ib-loopback/IB_write_524288_Avg_0:0": 23856.63,
        "ib-loopback/IB_write_1048576_Avg_0:0": 23869.58,
        "ib-loopback/IB_write_2097152_Avg_0:0": 23885.7,
        "ib-loopback/IB_write_4194304_Avg_0:0": 23914.56,
        "ib-loopback/IB_write_8388608_Avg_0:0": 23935.21,
        "ib-loopback/IB_write_512_Avg_1:0": 1479.1,
        "ib-loopback/IB_write_1024_Avg_1:0": 3246.48,
        "ib-loopback/IB_write_2048_Avg_1:0": 6754.94,
        "ib-loopback/IB_write_4096_Avg_1:0": 13101.09,
        "ib-loopback/IB_write_8192_Avg_1:0": 19945.81,
        "ib-loopback/IB_write_16384_Avg_1:0": 22631.62,
        "ib-loopback/IB_write_32768_Avg_1:0": 23381.58,
        "ib-loopback/IB_write_65536_Avg_1:0": 23515.29,
        "ib-loopback/IB_write_131072_Avg_1:0": 23756.01,
        "ib-loopback/IB_write_262144_Avg_1:0": 23875.91,
        "ib-loopback/IB_write_524288_Avg_1:0": 23911.13,
        "ib-loopback/IB_write_1048576_Avg_1:0": 23935.42,
        "ib-loopback/IB_write_2097152_Avg_1:0": 23941.25,
        "ib-loopback/IB_write_4194304_Avg_1:0": 23922.06,
        "ib-loopback/IB_write_8388608_Avg_1:0": 23928.59,
        "ib-loopback/IB_write_512_Avg_2:0": 1505.29,
        "ib-loopback/IB_write_1024_Avg_2:0": 3215.97,
        "ib-loopback/IB_write_2048_Avg_2:0": 6745.49,
        "ib-loopback/IB_write_4096_Avg_2:0": 11548.88,
        "ib-loopback/IB_write_8192_Avg_2:0": 19432.15,
        "ib-loopback/IB_write_16384_Avg_2:0": 22765.51,
        "ib-loopback/IB_write_32768_Avg_2:0": 23235.07,
        "ib-loopback/IB_write_65536_Avg_2:0": 23620.08,
        "ib-loopback/IB_write_131072_Avg_2:0": 23759.08,
        "ib-loopback/IB_write_262144_Avg_2:0": 23859.82,
        "ib-loopback/IB_write_524288_Avg_2:0": 23775.01,
        "ib-loopback/IB_write_1048576_Avg_2:0": 23885.96,
        "ib-loopback/IB_write_2097152_Avg_2:0": 23894.73,
        "ib-loopback/IB_write_4194304_Avg_2:0": 23909.98,
        "ib-loopback/IB_write_8388608_Avg_2:0": 23927.21,
        "ib-loopback/IB_write_512_Avg_3:0": 1474.47,
        "ib-loopback/IB_write_1024_Avg_3:0": 3344.68,
        "ib-loopback/IB_write_2048_Avg_3:0": 6606.35,
        "ib-loopback/IB_write_4096_Avg_3:0": 12071.4,
        "ib-loopback/IB_write_8192_Avg_3:0": 18408.33,
        "ib-loopback/IB_write_16384_Avg_3:0": 20789.27,
        "ib-loopback/IB_write_32768_Avg_3:0": 22469.43,
        "ib-loopback/IB_write_65536_Avg_3:0": 22777.1,
        "ib-loopback/IB_write_131072_Avg_3:0": 23461.03,
        "ib-loopback/IB_write_262144_Avg_3:0": 23397.19,
        "ib-loopback/IB_write_524288_Avg_3:0": 23526.55,
        "ib-loopback/IB_write_1048576_Avg_3:0": 23854.76,
        "ib-loopback/IB_write_2097152_Avg_3:0": 23862.23,
        "ib-loopback/IB_write_4194304_Avg_3:0": 23931.15,
        "ib-loopback/IB_write_8388608_Avg_3:0": 23924.44,
        "ib-loopback/IB_write_512_Avg_4:0": 1523.33,
        "ib-loopback/IB_write_1024_Avg_4:0": 3233.23,
        "ib-loopback/IB_write_2048_Avg_4:0": 6792.88,
        "ib-loopback/IB_write_4096_Avg_4:0": 12616.05,
        "ib-loopback/IB_write_8192_Avg_4:0": 19324.05,
        "ib-loopback/IB_write_16384_Avg_4:0": 22082.51,
        "ib-loopback/IB_write_32768_Avg_4:0": 23294.23,
        "ib-loopback/IB_write_65536_Avg_4:0": 23546.22,
        "ib-loopback/IB_write_131072_Avg_4:0": 23727.91,
        "ib-loopback/IB_write_262144_Avg_4:0": 23843.93,
        "ib-loopback/IB_write_524288_Avg_4:0": 23905.96,
        "ib-loopback/IB_write_1048576_Avg_4:0": 23902.37,
        "ib-loopback/IB_write_2097152_Avg_4:0": 23921.03,
        "ib-loopback/IB_write_4194304_Avg_4:0": 23921.06,
        "ib-loopback/IB_write_8388608_Avg_4:0": 23922.4,
        "ib-loopback/IB_write_512_Avg_5:0": 1506.63,
        "ib-loopback/IB_write_1024_Avg_5:0": 3261.71,
        "ib-loopback/IB_write_2048_Avg_5:0": 6752.38,
        "ib-loopback/IB_write_4096_Avg_5:0": 13592.41,
        "ib-loopback/IB_write_8192_Avg_5:0": 19989.31,
        "ib-loopback/IB_write_16384_Avg_5:0": 22560.57,
        "ib-loopback/IB_write_32768_Avg_5:0": 23389.29,
        "ib-loopback/IB_write_65536_Avg_5:0": 23503.32,
        "ib-loopback/IB_write_131072_Avg_5:0": 23741.51,
        "ib-loopback/IB_write_262144_Avg_5:0": 23866.43,
        "ib-loopback/IB_write_524288_Avg_5:0": 23898.95,
        "ib-loopback/IB_write_1048576_Avg_5:0": 23876.36,
        "ib-loopback/IB_write_2097152_Avg_5:0": 23919.63,
        "ib-loopback/IB_write_4194304_Avg_5:0": 23924.68,
        "ib-loopback/IB_write_8388608_Avg_5:0": 23930.37,
        "ib-loopback/IB_write_512_Avg_6:0": 1467.69,
        "ib-loopback/IB_write_1024_Avg_6:0": 3157.04,
        "ib-loopback/IB_write_2048_Avg_6:0": 6494.61,
        "ib-loopback/IB_write_4096_Avg_6:0": 12883.51,
        "ib-loopback/IB_write_8192_Avg_6:0": 19207.67,
        "ib-loopback/IB_write_16384_Avg_6:0": 22519.39,
        "ib-loopback/IB_write_32768_Avg_6:0": 23323.46,
        "ib-loopback/IB_write_65536_Avg_6:0": 23523.6,
        "ib-loopback/IB_write_131072_Avg_6:0": 23626.67,
        "ib-loopback/IB_write_262144_Avg_6:0": 23836.99,
        "ib-loopback/IB_write_524288_Avg_6:0": 23904.51,
        "ib-loopback/IB_write_1048576_Avg_6:0": 23919.07,
        "ib-loopback/IB_write_2097152_Avg_6:0": 23943.82,
        "ib-loopback/IB_write_4194304_Avg_6:0": 23936.77,
        "ib-loopback/IB_write_8388608_Avg_6:0": 23941.57,
        "ib-loopback/IB_write_512_Avg_7:0": 1505.63,
        "ib-loopback/IB_write_1024_Avg_7:0": 3259.93,
        "ib-loopback/IB_write_2048_Avg_7:0": 6738.6,
        "ib-loopback/IB_write_4096_Avg_7:0": 13352.06,
        "ib-loopback/IB_write_8192_Avg_7:0": 19941.35,
        "ib-loopback/IB_write_16384_Avg_7:0": 22566.09,
        "ib-loopback/IB_write_32768_Avg_7:0": 23244.77,
        "ib-loopback/IB_write_65536_Avg_7:0": 23377.67,
        "ib-loopback/IB_write_131072_Avg_7:0": 23736.17,
        "ib-loopback/IB_write_262144_Avg_7:0": 23829.25,
        "ib-loopback/IB_write_524288_Avg_7:0": 23879.6,
        "ib-loopback/IB_write_1048576_Avg_7:0": 23895.1,
        "ib-loopback/IB_write_2097152_Avg_7:0": 23930.64,
        "ib-loopback/IB_write_4194304_Avg_7:0": 23845.63,
        "ib-loopback/IB_write_8388608_Avg_7:0": 23896.94,
        "kernel-launch/return_code": 0.0,
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
        "kernel-launch/event_overhead:0": 0.00595,
        "kernel-launch/event_overhead:1": 0.00595,
        "kernel-launch/event_overhead:2": 0.00557,
        "kernel-launch/event_overhead:3": 0.0055,
        "kernel-launch/event_overhead:4": 0.00592,
        "kernel-launch/event_overhead:5": 0.00589,
        "kernel-launch/event_overhead:6": 0.00572,
        "kernel-launch/event_overhead:7": 0.0059,
        "kernel-launch/wall_overhead:0": 0.01026,
        "kernel-launch/wall_overhead:1": 0.01026,
        "kernel-launch/wall_overhead:2": 0.01046,
        "kernel-launch/wall_overhead:3": 0.01049,
        "kernel-launch/wall_overhead:4": 0.01063,
        "kernel-launch/wall_overhead:5": 0.01006,
        "kernel-launch/wall_overhead:6": 0.01045,
        "kernel-launch/wall_overhead:7": 0.01071,
781
782
783
784
785
786
        "lstm_models/pytorch-lstm/steptime_train_float32": 48.0702451896,
        "lstm_models/pytorch-lstm/throughput_train_float32": 4806.4724411328,
        "lstm_models/pytorch-lstm/steptime_train_float16": 25.9531298652,
        "lstm_models/pytorch-lstm/throughput_train_float16": 9069.9080925588,
        "pytorch-matmul/nosharding": 34.6449975967,
        "mem-bw/return_code": 0.0,
787
788
789
790
791
792
793
794
        "mem-bw/H2D_Mem_BW:0": 25.6,
        "mem-bw/H2D_Mem_BW:1": 25.8,
        "mem-bw/H2D_Mem_BW:2": 26.0,
        "mem-bw/H2D_Mem_BW:3": 26.1,
        "mem-bw/H2D_Mem_BW:4": 26.2,
        "mem-bw/H2D_Mem_BW:5": 25.8,
        "mem-bw/H2D_Mem_BW:6": 25.3,
        "mem-bw/H2D_Mem_BW:7": 26.1,
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
        "mem-bw/D2H_Mem_BW:0": 24.3,
        "mem-bw/D2H_Mem_BW:1": 24.6,
        "mem-bw/D2H_Mem_BW:2": 24.5,
        "mem-bw/D2H_Mem_BW:3": 24.6,
        "mem-bw/D2H_Mem_BW:4": 24.3,
        "mem-bw/D2H_Mem_BW:5": 24.3,
        "mem-bw/D2H_Mem_BW:6": 23.9,
        "mem-bw/D2H_Mem_BW:7": 24.6,
        "mem-bw/D2D_Mem_BW:0": 1118.0,
        "mem-bw/D2D_Mem_BW:1": 1114.6,
        "mem-bw/D2D_Mem_BW:2": 1119.7,
        "mem-bw/D2D_Mem_BW:3": 1121.9,
        "mem-bw/D2D_Mem_BW:4": 1109.7,
        "mem-bw/D2D_Mem_BW:5": 1110.1,
        "mem-bw/D2D_Mem_BW:6": 1123.3,
        "mem-bw/D2D_Mem_BW:7": 1117.6,
        "nccl-bw/allreduce_8_busbw:0": 0.0,
        "nccl-bw/allreduce_8_algbw:0": 0.0,
        "nccl-bw/allreduce_8_time:0": 37.84,
        "nccl-bw/allreduce_16_busbw:0": 0.0,
        "nccl-bw/allreduce_16_algbw:0": 0.0,
        "nccl-bw/allreduce_16_time:0": 36.42,
        "nccl-bw/allreduce_32_busbw:0": 0.0,
        "nccl-bw/allreduce_32_algbw:0": 0.0,
        "nccl-bw/allreduce_32_time:0": 36.87,
        "nccl-bw/allreduce_64_busbw:0": 0.0,
        "nccl-bw/allreduce_64_algbw:0": 0.0,
        "nccl-bw/allreduce_64_time:0": 35.83,
        "nccl-bw/allreduce_128_busbw:0": 0.01,
        "nccl-bw/allreduce_128_algbw:0": 0.0,
        "nccl-bw/allreduce_128_time:0": 36.91,
        "nccl-bw/allreduce_256_busbw:0": 0.01,
        "nccl-bw/allreduce_256_algbw:0": 0.01,
        "nccl-bw/allreduce_256_time:0": 37.58,
        "nccl-bw/allreduce_512_busbw:0": 0.02,
        "nccl-bw/allreduce_512_algbw:0": 0.01,
        "nccl-bw/allreduce_512_time:0": 36.98,
        "nccl-bw/allreduce_1024_busbw:0": 0.05,
        "nccl-bw/allreduce_1024_algbw:0": 0.03,
        "nccl-bw/allreduce_1024_time:0": 36.93,
        "nccl-bw/allreduce_2048_busbw:0": 0.1,
        "nccl-bw/allreduce_2048_algbw:0": 0.06,
        "nccl-bw/allreduce_2048_time:0": 36.06,
        "nccl-bw/allreduce_4096_busbw:0": 0.19,
        "nccl-bw/allreduce_4096_algbw:0": 0.11,
        "nccl-bw/allreduce_4096_time:0": 37.2,
        "nccl-bw/allreduce_8192_busbw:0": 0.39,
        "nccl-bw/allreduce_8192_algbw:0": 0.22,
        "nccl-bw/allreduce_8192_time:0": 37.04,
        "nccl-bw/allreduce_16384_busbw:0": 0.77,
        "nccl-bw/allreduce_16384_algbw:0": 0.44,
        "nccl-bw/allreduce_16384_time:0": 37.46,
        "nccl-bw/allreduce_32768_busbw:0": 1.52,
        "nccl-bw/allreduce_32768_algbw:0": 0.87,
        "nccl-bw/allreduce_32768_time:0": 37.64,
        "nccl-bw/allreduce_65536_busbw:0": 3.0,
        "nccl-bw/allreduce_65536_algbw:0": 1.71,
        "nccl-bw/allreduce_65536_time:0": 38.22,
        "nccl-bw/allreduce_131072_busbw:0": 5.31,
        "nccl-bw/allreduce_131072_algbw:0": 3.04,
        "nccl-bw/allreduce_131072_time:0": 43.17,
        "nccl-bw/allreduce_262144_busbw:0": 9.5,
        "nccl-bw/allreduce_262144_algbw:0": 5.43,
        "nccl-bw/allreduce_262144_time:0": 48.29,
        "nccl-bw/allreduce_524288_busbw:0": 15.11,
        "nccl-bw/allreduce_524288_algbw:0": 8.64,
        "nccl-bw/allreduce_524288_time:0": 60.71,
        "nccl-bw/allreduce_1048576_busbw:0": 24.1,
        "nccl-bw/allreduce_1048576_algbw:0": 13.77,
        "nccl-bw/allreduce_1048576_time:0": 76.13,
        "nccl-bw/allreduce_2097152_busbw:0": 38.12,
        "nccl-bw/allreduce_2097152_algbw:0": 21.78,
        "nccl-bw/allreduce_2097152_time:0": 96.28,
        "nccl-bw/allreduce_4194304_busbw:0": 65.75,
        "nccl-bw/allreduce_4194304_algbw:0": 37.57,
        "nccl-bw/allreduce_4194304_time:0": 111.6,
        "nccl-bw/allreduce_8388608_busbw:0": 89.51,
        "nccl-bw/allreduce_8388608_algbw:0": 51.15,
        "nccl-bw/allreduce_8388608_time:0": 164.0,
        "nccl-bw/allreduce_16777216_busbw:0": 114.38,
        "nccl-bw/allreduce_16777216_algbw:0": 65.36,
        "nccl-bw/allreduce_16777216_time:0": 256.7,
        "nccl-bw/allreduce_33554432_busbw:0": 154.89,
        "nccl-bw/allreduce_33554432_algbw:0": 88.51,
        "nccl-bw/allreduce_33554432_time:0": 379.1,
        "nccl-bw/allreduce_67108864_busbw:0": 200.01,
        "nccl-bw/allreduce_67108864_algbw:0": 114.29,
        "nccl-bw/allreduce_67108864_time:0": 587.2,
        "nccl-bw/allreduce_134217728_busbw:0": 202.97,
        "nccl-bw/allreduce_134217728_algbw:0": 115.98,
        "nccl-bw/allreduce_134217728_time:0": 1157.2,
        "nccl-bw/allreduce_268435456_busbw:0": 221.82,
        "nccl-bw/allreduce_268435456_algbw:0": 126.75,
        "nccl-bw/allreduce_268435456_time:0": 2117.8,
        "nccl-bw/allreduce_536870912_busbw:0": 224.54,
        "nccl-bw/allreduce_536870912_algbw:0": 128.31,
        "nccl-bw/allreduce_536870912_time:0": 4184.2,
        "nccl-bw/allreduce_1073741824_busbw:0": 230.15,
        "nccl-bw/allreduce_1073741824_algbw:0": 131.51,
        "nccl-bw/allreduce_1073741824_time:0": 8164.5,
        "nccl-bw/allreduce_2147483648_busbw:0": 231.89,
        "nccl-bw/allreduce_2147483648_algbw:0": 132.51,
        "nccl-bw/allreduce_2147483648_time:0": 16207.0,
        "nccl-bw/allreduce_4294967296_busbw:0": 234.45,
        "nccl-bw/allreduce_4294967296_algbw:0": 133.97,
        "nccl-bw/allreduce_4294967296_time:0": 32059.0,
        "nccl-bw/allreduce_8589934592_busbw:0": 235.36,
        "nccl-bw/allreduce_8589934592_algbw:0": 134.49,
        "nccl-bw/allreduce_8589934592_time:0": 63870.0,
        "resnet_models/pytorch-resnet50/steptime_train_float32": 253.9552273229,
        "resnet_models/pytorch-resnet50/throughput_train_float32": 760.334809913,
        "resnet_models/pytorch-resnet50/steptime_train_float16": 200.0860618427,
        "resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430923,
        "resnet_models/pytorch-resnet101/steptime_train_float32": 389.0860509127,
        "resnet_models/pytorch-resnet101/throughput_train_float32": 496.117474093,
        "resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918,
        "resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195,
        "resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278315,
        "resnet_models/pytorch-resnet152/throughput_train_float32": 352.0709954335,
        "resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802,
        "resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998154,
        "pytorch-sharding-matmul/allreduce": 10.574411869,
        "pytorch-sharding-matmul/allgather": 10.0846967697,
        "vgg_models/pytorch-vgg11/steptime_train_float32": 40.3528367169,
        "vgg_models/pytorch-vgg11/throughput_train_float32": 796.361593695,
        "vgg_models/pytorch-vgg11/steptime_train_float16": 24.1335148457,
        "vgg_models/pytorch-vgg11/throughput_train_float16": 1330.4113614585,
        "vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936,
        "vgg_models/pytorch-vgg13/throughput_train_float32": 580.2341074444,
        "vgg_models/pytorch-vgg13/steptime_train_float16": 33.3522899309,
        "vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023902,
        "vgg_models/pytorch-vgg16/steptime_train_float32": 65.225199447,
        "vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638876,
        "vgg_models/pytorch-vgg16/steptime_train_float16": 39.2528773518,
        "vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546148,
        "vgg_models/pytorch-vgg19/steptime_train_float32": 74.9348710524,
        "vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
        "vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
        "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377,
934
935
936
937
        "diagnosis/accept": true,
        "diagnosis/issue_num": 0,
        "diagnosis/category": "",
        "diagnosis/issue_details": "",
938
939
940
        "Index": "sb-validation-02"
    },
    {
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
        "bert_models/pytorch-bert-base/steptime_train_float32": 114.5916701062,
        "bert_models/pytorch-bert-base/throughput_train_float32": 279.8794623591,
        "bert_models/pytorch-bert-base/steptime_train_float16": 83.8895108318,
        "bert_models/pytorch-bert-base/throughput_train_float16": 382.0672582742,
        "bert_models/pytorch-bert-large/steptime_train_float32": 307.9359371914,
        "bert_models/pytorch-bert-large/throughput_train_float32": 103.9487609742,
        "bert_models/pytorch-bert-large/steptime_train_float16": 206.8114168942,
        "bert_models/pytorch-bert-large/throughput_train_float16": 154.8408911711,
        "pytorch-computation-communication-overlap/mul_cost:0": 44.1822062144,
        "pytorch-computation-communication-overlap/mul_cost:1": 44.1822139389,
        "pytorch-computation-communication-overlap/mul_cost:2": 43.9701470781,
        "pytorch-computation-communication-overlap/mul_cost:3": 43.9701478756,
        "pytorch-computation-communication-overlap/mul_cost:4": 43.9701779317,
        "pytorch-computation-communication-overlap/mul_cost:5": 43.9701571606,
        "pytorch-computation-communication-overlap/mul_cost:6": 43.9701651983,
        "pytorch-computation-communication-overlap/mul_cost:7": 44.1795444785,
        "pytorch-computation-communication-overlap/matmul_cost:0": 137.0477370556,
        "pytorch-computation-communication-overlap/matmul_cost:1": 137.0478344693,
        "pytorch-computation-communication-overlap/matmul_cost:2": 137.047772209,
        "pytorch-computation-communication-overlap/matmul_cost:3": 137.0477969726,
        "pytorch-computation-communication-overlap/matmul_cost:4": 137.0481367431,
        "pytorch-computation-communication-overlap/matmul_cost:5": 137.0482198877,
        "pytorch-computation-communication-overlap/matmul_cost:6": 137.0477532237,
        "pytorch-computation-communication-overlap/matmul_cost:7": 137.0478081607,
        "densenet_models/pytorch-densenet169/steptime_train_float32": 150.6415554322,
        "densenet_models/pytorch-densenet169/throughput_train_float32": 212.4701719243,
        "densenet_models/pytorch-densenet169/steptime_train_float16": 145.3081957297,
        "densenet_models/pytorch-densenet169/throughput_train_float16": 220.2932337305,
        "densenet_models/pytorch-densenet201/steptime_train_float32": 182.9111778643,
        "densenet_models/pytorch-densenet201/throughput_train_float32": 174.9742278232,
        "densenet_models/pytorch-densenet201/steptime_train_float16": 176.3489063596,
        "densenet_models/pytorch-densenet201/throughput_train_float16": 181.5446573603,
        "gemm-flops/FP64:0": 9031.23,
        "gemm-flops/FP64:1": 9040.85,
        "gemm-flops/FP64:2": 9010.56,
        "gemm-flops/FP64:3": 9041.26,
        "gemm-flops/FP64:4": 9039.19,
        "gemm-flops/FP64:5": 9015.69,
        "gemm-flops/FP64:6": 9022.19,
        "gemm-flops/FP64:7": 9030.2,
        "gemm-flops/FP32:0": 18362.1,
        "gemm-flops/FP32:1": 18375.6,
        "gemm-flops/FP32:2": 18314.9,
        "gemm-flops/FP32:3": 18375.6,
        "gemm-flops/FP32:4": 18368.6,
        "gemm-flops/FP32:5": 18347.1,
        "gemm-flops/FP32:6": 18247.4,
        "gemm-flops/FP32:7": 18318.4,
        "gemm-flops/FP16:0": 33878.0,
        "gemm-flops/FP16:1": 33911.1,
        "gemm-flops/FP16:2": 33769.3,
        "gemm-flops/FP16:3": 33909.9,
        "gemm-flops/FP16:4": 33896.5,
        "gemm-flops/FP16:5": 33798.1,
        "gemm-flops/FP16:6": 33647.3,
        "gemm-flops/FP16:7": 33764.8,
        "gemm-flops/FP64_TC:0": 18963.6,
        "gemm-flops/FP64_TC:1": 18924.2,
        "gemm-flops/FP64_TC:2": 18930.3,
        "gemm-flops/FP64_TC:3": 18971.9,
        "gemm-flops/FP64_TC:4": 18946.0,
        "gemm-flops/FP64_TC:5": 18945.0,
        "gemm-flops/FP64_TC:6": 18822.9,
        "gemm-flops/FP64_TC:7": 18911.1,
        "gemm-flops/TF32_TC:0": 127900.0,
        "gemm-flops/TF32_TC:1": 129094.0,
        "gemm-flops/TF32_TC:2": 127831.0,
        "gemm-flops/TF32_TC:3": 128709.0,
        "gemm-flops/TF32_TC:4": 127388.0,
        "gemm-flops/TF32_TC:5": 127861.0,
        "gemm-flops/TF32_TC:6": 128492.0,
        "gemm-flops/TF32_TC:7": 127720.0,
        "gemm-flops/BF16_TC:0": 264965.0,
        "gemm-flops/BF16_TC:1": 266638.0,
        "gemm-flops/BF16_TC:2": 263151.0,
        "gemm-flops/BF16_TC:3": 264752.0,
        "gemm-flops/BF16_TC:4": 263049.0,
        "gemm-flops/BF16_TC:5": 266605.0,
        "gemm-flops/BF16_TC:6": 267501.0,
        "gemm-flops/BF16_TC:7": 263880.0,
        "gemm-flops/FP16_TC:0": 279474.0,
        "gemm-flops/FP16_TC:1": 281256.0,
        "gemm-flops/FP16_TC:2": 277403.0,
        "gemm-flops/FP16_TC:3": 279147.0,
        "gemm-flops/FP16_TC:4": 277587.0,
        "gemm-flops/FP16_TC:5": 281537.0,
        "gemm-flops/FP16_TC:6": 282132.0,
        "gemm-flops/FP16_TC:7": 277788.0,
        "gemm-flops/INT8_TC:0": 475160.0,
        "gemm-flops/INT8_TC:1": 477725.0,
        "gemm-flops/INT8_TC:2": 471621.0,
        "gemm-flops/INT8_TC:3": 473716.0,
        "gemm-flops/INT8_TC:4": 472124.0,
        "gemm-flops/INT8_TC:5": 479972.0,
        "gemm-flops/INT8_TC:6": 481327.0,
        "gemm-flops/INT8_TC:7": 474710.0,
        "gemm-flops/INT4_TC:0": 970330.0,
        "gemm-flops/INT4_TC:1": 976837.0,
        "gemm-flops/INT4_TC:2": 966003.0,
        "gemm-flops/INT4_TC:3": 971315.0,
        "gemm-flops/INT4_TC:4": 964441.0,
        "gemm-flops/INT4_TC:5": 982461.0,
        "gemm-flops/INT4_TC:6": 979610.0,
        "gemm-flops/INT4_TC:7": 968359.0,
        "gpt_models/pytorch-gpt2-large/steptime_train_float32": 295.0526971836,
        "gpt_models/pytorch-gpt2-large/throughput_train_float32": 27.1154543969,
        "gpt_models/pytorch-gpt2-large/steptime_train_float16": 194.4957742235,
        "gpt_models/pytorch-gpt2-large/throughput_train_float16": 41.1394499411,
        "gpu-sm-copy-bw/dtoh:0": 3.91755,
        "gpu-sm-copy-bw/dtoh:1": 4.45414,
        "gpu-sm-copy-bw/dtoh:2": 1.26483,
        "gpu-sm-copy-bw/dtoh:3": 1.30041,
        "gpu-sm-copy-bw/dtoh:4": 1.31577,
        "gpu-sm-copy-bw/dtoh:5": 1.27968,
        "gpu-sm-copy-bw/dtoh:6": 4.47849,
        "gpu-sm-copy-bw/dtoh:7": 3.96231,
        "gpu-sm-copy-bw/dtoh:8": 3.91705,
        "gpu-sm-copy-bw/dtoh:9": 4.45487,
        "gpu-sm-copy-bw/dtoh:10": 1.26352,
        "gpu-sm-copy-bw/dtoh:11": 1.2999,
        "gpu-sm-copy-bw/dtoh:12": 1.31677,
        "gpu-sm-copy-bw/dtoh:13": 1.27885,
        "gpu-sm-copy-bw/dtoh:14": 4.47913,
        "gpu-sm-copy-bw/dtoh:15": 3.95893,
        "gpu-sm-copy-bw/dtoh:16": 3.91729,
        "gpu-sm-copy-bw/dtoh:17": 4.45627,
        "gpu-sm-copy-bw/dtoh:18": 1.26437,
        "gpu-sm-copy-bw/dtoh:19": 1.30144,
        "gpu-sm-copy-bw/dtoh:20": 1.31704,
        "gpu-sm-copy-bw/dtoh:21": 1.27857,
        "gpu-sm-copy-bw/dtoh:22": 4.47889,
        "gpu-sm-copy-bw/dtoh:23": 3.95984,
        "gpu-sm-copy-bw/dtoh:24": 3.92025,
        "gpu-sm-copy-bw/dtoh:25": 4.45423,
        "gpu-sm-copy-bw/dtoh:26": 1.26449,
        "gpu-sm-copy-bw/dtoh:27": 1.29954,
        "gpu-sm-copy-bw/dtoh:28": 1.31731,
        "gpu-sm-copy-bw/dtoh:29": 1.27916,
        "gpu-sm-copy-bw/dtoh:30": 4.4797,
        "gpu-sm-copy-bw/dtoh:31": 3.96124,
        "gpu-sm-copy-bw/htod:0": 23.9685,
        "gpu-sm-copy-bw/htod:1": 23.967,
        "gpu-sm-copy-bw/htod:2": 19.9898,
        "gpu-sm-copy-bw/htod:3": 20.2848,
        "gpu-sm-copy-bw/htod:4": 20.3931,
        "gpu-sm-copy-bw/htod:5": 20.0888,
        "gpu-sm-copy-bw/htod:6": 23.9766,
        "gpu-sm-copy-bw/htod:7": 23.9792,
        "gpu-sm-copy-bw/htod:8": 23.9497,
        "gpu-sm-copy-bw/htod:9": 23.9438,
        "gpu-sm-copy-bw/htod:10": 20.0039,
        "gpu-sm-copy-bw/htod:11": 20.2469,
        "gpu-sm-copy-bw/htod:12": 20.4344,
        "gpu-sm-copy-bw/htod:13": 20.1005,
        "gpu-sm-copy-bw/htod:14": 23.9491,
        "gpu-sm-copy-bw/htod:15": 23.9898,
        "gpu-sm-copy-bw/htod:16": 23.962,
        "gpu-sm-copy-bw/htod:17": 23.97,
        "gpu-sm-copy-bw/htod:18": 19.987,
        "gpu-sm-copy-bw/htod:19": 20.1549,
        "gpu-sm-copy-bw/htod:20": 20.3931,
        "gpu-sm-copy-bw/htod:21": 20.1013,
        "gpu-sm-copy-bw/htod:22": 23.9865,
        "gpu-sm-copy-bw/htod:23": 23.972,
        "gpu-sm-copy-bw/htod:24": 23.9569,
        "gpu-sm-copy-bw/htod:25": 23.9762,
        "gpu-sm-copy-bw/htod:26": 19.92,
        "gpu-sm-copy-bw/htod:27": 20.2638,
        "gpu-sm-copy-bw/htod:28": 20.4419,
        "gpu-sm-copy-bw/htod:29": 20.1054,
        "gpu-sm-copy-bw/htod:30": 23.9752,
        "gpu-sm-copy-bw/htod:31": 23.9631,
        "ib-loopback/IB_write_512_Avg_0:0": 1492.89,
        "ib-loopback/IB_write_1024_Avg_0:0": 3224.92,
        "ib-loopback/IB_write_2048_Avg_0:0": 6714.5,
        "ib-loopback/IB_write_4096_Avg_0:0": 12871.93,
        "ib-loopback/IB_write_8192_Avg_0:0": 19990.78,
        "ib-loopback/IB_write_16384_Avg_0:0": 22172.25,
        "ib-loopback/IB_write_32768_Avg_0:0": 23073.19,
        "ib-loopback/IB_write_65536_Avg_0:0": 23527.09,
        "ib-loopback/IB_write_131072_Avg_0:0": 23805.92,
        "ib-loopback/IB_write_262144_Avg_0:0": 23380.51,
        "ib-loopback/IB_write_524288_Avg_0:0": 23856.63,
        "ib-loopback/IB_write_1048576_Avg_0:0": 23869.58,
        "ib-loopback/IB_write_2097152_Avg_0:0": 23885.7,
        "ib-loopback/IB_write_4194304_Avg_0:0": 23914.56,
        "ib-loopback/IB_write_8388608_Avg_0:0": 23935.21,
        "ib-loopback/IB_write_512_Avg_1:0": 1479.1,
        "ib-loopback/IB_write_1024_Avg_1:0": 3246.48,
        "ib-loopback/IB_write_2048_Avg_1:0": 6754.94,
        "ib-loopback/IB_write_4096_Avg_1:0": 13101.09,
        "ib-loopback/IB_write_8192_Avg_1:0": 19945.81,
        "ib-loopback/IB_write_16384_Avg_1:0": 22631.62,
        "ib-loopback/IB_write_32768_Avg_1:0": 23381.58,
        "ib-loopback/IB_write_65536_Avg_1:0": 23515.29,
        "ib-loopback/IB_write_131072_Avg_1:0": 23756.01,
        "ib-loopback/IB_write_262144_Avg_1:0": 23875.91,
        "ib-loopback/IB_write_524288_Avg_1:0": 23911.13,
        "ib-loopback/IB_write_1048576_Avg_1:0": 23935.42,
        "ib-loopback/IB_write_2097152_Avg_1:0": 23941.25,
        "ib-loopback/IB_write_4194304_Avg_1:0": 23922.06,
        "ib-loopback/IB_write_8388608_Avg_1:0": 23928.59,
        "ib-loopback/IB_write_512_Avg_2:0": 1505.29,
        "ib-loopback/IB_write_1024_Avg_2:0": 3215.97,
        "ib-loopback/IB_write_2048_Avg_2:0": 6745.49,
        "ib-loopback/IB_write_4096_Avg_2:0": 11548.88,
        "ib-loopback/IB_write_8192_Avg_2:0": 19432.15,
        "ib-loopback/IB_write_16384_Avg_2:0": 22765.51,
        "ib-loopback/IB_write_32768_Avg_2:0": 23235.07,
        "ib-loopback/IB_write_65536_Avg_2:0": 23620.08,
        "ib-loopback/IB_write_131072_Avg_2:0": 23759.08,
        "ib-loopback/IB_write_262144_Avg_2:0": 23859.82,
        "ib-loopback/IB_write_524288_Avg_2:0": 23775.01,
        "ib-loopback/IB_write_1048576_Avg_2:0": 23885.96,
        "ib-loopback/IB_write_2097152_Avg_2:0": 23894.73,
        "ib-loopback/IB_write_4194304_Avg_2:0": 23909.98,
        "ib-loopback/IB_write_8388608_Avg_2:0": 23927.21,
        "ib-loopback/IB_write_512_Avg_3:0": 1474.47,
        "ib-loopback/IB_write_1024_Avg_3:0": 3344.68,
        "ib-loopback/IB_write_2048_Avg_3:0": 6606.35,
        "ib-loopback/IB_write_4096_Avg_3:0": 12071.4,
        "ib-loopback/IB_write_8192_Avg_3:0": 18408.33,
        "ib-loopback/IB_write_16384_Avg_3:0": 20789.27,
        "ib-loopback/IB_write_32768_Avg_3:0": 22469.43,
        "ib-loopback/IB_write_65536_Avg_3:0": 22777.1,
        "ib-loopback/IB_write_131072_Avg_3:0": 23461.03,
        "ib-loopback/IB_write_262144_Avg_3:0": 23397.19,
        "ib-loopback/IB_write_524288_Avg_3:0": 23526.55,
        "ib-loopback/IB_write_1048576_Avg_3:0": 23854.76,
        "ib-loopback/IB_write_2097152_Avg_3:0": 23862.23,
        "ib-loopback/IB_write_4194304_Avg_3:0": 23931.15,
        "ib-loopback/IB_write_8388608_Avg_3:0": 23924.44,
        "ib-loopback/IB_write_512_Avg_4:0": 1523.33,
        "ib-loopback/IB_write_1024_Avg_4:0": 3233.23,
        "ib-loopback/IB_write_2048_Avg_4:0": 6792.88,
        "ib-loopback/IB_write_4096_Avg_4:0": 12616.05,
        "ib-loopback/IB_write_8192_Avg_4:0": 19324.05,
        "ib-loopback/IB_write_16384_Avg_4:0": 22082.51,
        "ib-loopback/IB_write_32768_Avg_4:0": 23294.23,
        "ib-loopback/IB_write_65536_Avg_4:0": 23546.22,
        "ib-loopback/IB_write_131072_Avg_4:0": 23727.91,
        "ib-loopback/IB_write_262144_Avg_4:0": 23843.93,
        "ib-loopback/IB_write_524288_Avg_4:0": 23905.96,
        "ib-loopback/IB_write_1048576_Avg_4:0": 23902.37,
        "ib-loopback/IB_write_2097152_Avg_4:0": 23921.03,
        "ib-loopback/IB_write_4194304_Avg_4:0": 23921.06,
        "ib-loopback/IB_write_8388608_Avg_4:0": 23922.4,
        "ib-loopback/IB_write_512_Avg_5:0": 1506.63,
        "ib-loopback/IB_write_1024_Avg_5:0": 3261.71,
        "ib-loopback/IB_write_2048_Avg_5:0": 6752.38,
        "ib-loopback/IB_write_4096_Avg_5:0": 13592.41,
        "ib-loopback/IB_write_8192_Avg_5:0": 19989.31,
        "ib-loopback/IB_write_16384_Avg_5:0": 22560.57,
        "ib-loopback/IB_write_32768_Avg_5:0": 23389.29,
        "ib-loopback/IB_write_65536_Avg_5:0": 23503.32,
        "ib-loopback/IB_write_131072_Avg_5:0": 23741.51,
        "ib-loopback/IB_write_262144_Avg_5:0": 23866.43,
        "ib-loopback/IB_write_524288_Avg_5:0": 23898.95,
        "ib-loopback/IB_write_1048576_Avg_5:0": 23876.36,
        "ib-loopback/IB_write_2097152_Avg_5:0": 23919.63,
        "ib-loopback/IB_write_4194304_Avg_5:0": 23924.68,
        "ib-loopback/IB_write_8388608_Avg_5:0": 23930.37,
        "ib-loopback/IB_write_512_Avg_6:0": 1467.69,
        "ib-loopback/IB_write_1024_Avg_6:0": 3157.04,
        "ib-loopback/IB_write_2048_Avg_6:0": 6494.61,
        "ib-loopback/IB_write_4096_Avg_6:0": 12883.51,
        "ib-loopback/IB_write_8192_Avg_6:0": 19207.67,
        "ib-loopback/IB_write_16384_Avg_6:0": 22519.39,
        "ib-loopback/IB_write_32768_Avg_6:0": 23323.46,
        "ib-loopback/IB_write_65536_Avg_6:0": 23523.6,
        "ib-loopback/IB_write_131072_Avg_6:0": 23626.67,
        "ib-loopback/IB_write_262144_Avg_6:0": 23836.99,
        "ib-loopback/IB_write_524288_Avg_6:0": 23904.51,
        "ib-loopback/IB_write_1048576_Avg_6:0": 23919.07,
        "ib-loopback/IB_write_2097152_Avg_6:0": 23943.82,
        "ib-loopback/IB_write_4194304_Avg_6:0": 23936.77,
        "ib-loopback/IB_write_8388608_Avg_6:0": 23941.57,
        "ib-loopback/IB_write_512_Avg_7:0": 1505.63,
        "ib-loopback/IB_write_1024_Avg_7:0": 3259.93,
        "ib-loopback/IB_write_2048_Avg_7:0": 6738.6,
        "ib-loopback/IB_write_4096_Avg_7:0": 13352.06,
        "ib-loopback/IB_write_8192_Avg_7:0": 19941.35,
        "ib-loopback/IB_write_16384_Avg_7:0": 22566.09,
        "ib-loopback/IB_write_32768_Avg_7:0": 23244.77,
        "ib-loopback/IB_write_65536_Avg_7:0": 23377.67,
        "ib-loopback/IB_write_131072_Avg_7:0": 23736.17,
        "ib-loopback/IB_write_262144_Avg_7:0": 23829.25,
        "ib-loopback/IB_write_524288_Avg_7:0": 23879.6,
        "ib-loopback/IB_write_1048576_Avg_7:0": 23895.1,
        "ib-loopback/IB_write_2097152_Avg_7:0": 23930.64,
        "ib-loopback/IB_write_4194304_Avg_7:0": 23845.63,
        "ib-loopback/IB_write_8388608_Avg_7:0": 23896.94,
        "kernel-launch/return_code": 0.0,
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
        "kernel-launch/event_overhead:0": 0.00596,
        "kernel-launch/event_overhead:1": 0.00595,
        "kernel-launch/event_overhead:2": 0.00557,
        "kernel-launch/event_overhead:3": 0.0055,
        "kernel-launch/event_overhead:4": 0.00592,
        "kernel-launch/event_overhead:5": 0.00589,
        "kernel-launch/event_overhead:6": 0.00572,
        "kernel-launch/event_overhead:7": 0.0059,
        "kernel-launch/wall_overhead:0": 0.01026,
        "kernel-launch/wall_overhead:1": 0.01026,
        "kernel-launch/wall_overhead:2": 0.01046,
        "kernel-launch/wall_overhead:3": 0.01049,
        "kernel-launch/wall_overhead:4": 0.01063,
        "kernel-launch/wall_overhead:5": 0.01006,
        "kernel-launch/wall_overhead:6": 0.01045,
        "kernel-launch/wall_overhead:7": 0.01071,
1250
1251
1252
1253
1254
1255
        "lstm_models/pytorch-lstm/steptime_train_float32": 48.0702451896,
        "lstm_models/pytorch-lstm/throughput_train_float32": 4806.4724411328,
        "lstm_models/pytorch-lstm/steptime_train_float16": 25.9531298652,
        "lstm_models/pytorch-lstm/throughput_train_float16": 9069.9080925588,
        "pytorch-matmul/nosharding": 34.6449975967,
        "mem-bw/return_code": 1.0,
1256
1257
1258
1259
1260
1261
1262
1263
        "mem-bw/H2D_Mem_BW:0": "",
        "mem-bw/H2D_Mem_BW:1": "",
        "mem-bw/H2D_Mem_BW:2": "",
        "mem-bw/H2D_Mem_BW:3": "",
        "mem-bw/H2D_Mem_BW:4": "",
        "mem-bw/H2D_Mem_BW:5": "",
        "mem-bw/H2D_Mem_BW:6": "",
        "mem-bw/H2D_Mem_BW:7": "",
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
        "mem-bw/D2H_Mem_BW:0": "",
        "mem-bw/D2H_Mem_BW:1": "",
        "mem-bw/D2H_Mem_BW:2": "",
        "mem-bw/D2H_Mem_BW:3": "",
        "mem-bw/D2H_Mem_BW:4": "",
        "mem-bw/D2H_Mem_BW:5": "",
        "mem-bw/D2H_Mem_BW:6": "",
        "mem-bw/D2H_Mem_BW:7": "",
        "mem-bw/D2D_Mem_BW:0": "",
        "mem-bw/D2D_Mem_BW:1": "",
        "mem-bw/D2D_Mem_BW:2": "",
        "mem-bw/D2D_Mem_BW:3": "",
        "mem-bw/D2D_Mem_BW:4": "",
        "mem-bw/D2D_Mem_BW:5": "",
        "mem-bw/D2D_Mem_BW:6": "",
        "mem-bw/D2D_Mem_BW:7": "",
        "nccl-bw/allreduce_8_busbw:0": 0.0,
        "nccl-bw/allreduce_8_algbw:0": 0.0,
        "nccl-bw/allreduce_8_time:0": 37.84,
        "nccl-bw/allreduce_16_busbw:0": 0.0,
        "nccl-bw/allreduce_16_algbw:0": 0.0,
        "nccl-bw/allreduce_16_time:0": 36.42,
        "nccl-bw/allreduce_32_busbw:0": 0.0,
        "nccl-bw/allreduce_32_algbw:0": 0.0,
        "nccl-bw/allreduce_32_time:0": 36.87,
        "nccl-bw/allreduce_64_busbw:0": 0.0,
        "nccl-bw/allreduce_64_algbw:0": 0.0,
        "nccl-bw/allreduce_64_time:0": 35.83,
        "nccl-bw/allreduce_128_busbw:0": 0.01,
        "nccl-bw/allreduce_128_algbw:0": 0.0,
        "nccl-bw/allreduce_128_time:0": 36.91,
        "nccl-bw/allreduce_256_busbw:0": 0.01,
        "nccl-bw/allreduce_256_algbw:0": 0.01,
        "nccl-bw/allreduce_256_time:0": 37.58,
        "nccl-bw/allreduce_512_busbw:0": 0.02,
        "nccl-bw/allreduce_512_algbw:0": 0.01,
        "nccl-bw/allreduce_512_time:0": 36.98,
        "nccl-bw/allreduce_1024_busbw:0": 0.05,
        "nccl-bw/allreduce_1024_algbw:0": 0.03,
        "nccl-bw/allreduce_1024_time:0": 36.93,
        "nccl-bw/allreduce_2048_busbw:0": 0.1,
        "nccl-bw/allreduce_2048_algbw:0": 0.06,
        "nccl-bw/allreduce_2048_time:0": 36.06,
        "nccl-bw/allreduce_4096_busbw:0": 0.19,
        "nccl-bw/allreduce_4096_algbw:0": 0.11,
        "nccl-bw/allreduce_4096_time:0": 37.2,
        "nccl-bw/allreduce_8192_busbw:0": 0.39,
        "nccl-bw/allreduce_8192_algbw:0": 0.22,
        "nccl-bw/allreduce_8192_time:0": 37.04,
        "nccl-bw/allreduce_16384_busbw:0": 0.77,
        "nccl-bw/allreduce_16384_algbw:0": 0.44,
        "nccl-bw/allreduce_16384_time:0": 37.46,
        "nccl-bw/allreduce_32768_busbw:0": 1.52,
        "nccl-bw/allreduce_32768_algbw:0": 0.87,
        "nccl-bw/allreduce_32768_time:0": 37.64,
        "nccl-bw/allreduce_65536_busbw:0": 3.0,
        "nccl-bw/allreduce_65536_algbw:0": 1.71,
        "nccl-bw/allreduce_65536_time:0": 38.22,
        "nccl-bw/allreduce_131072_busbw:0": 5.31,
        "nccl-bw/allreduce_131072_algbw:0": 3.04,
        "nccl-bw/allreduce_131072_time:0": 43.17,
        "nccl-bw/allreduce_262144_busbw:0": 9.5,
        "nccl-bw/allreduce_262144_algbw:0": 5.43,
        "nccl-bw/allreduce_262144_time:0": 48.29,
        "nccl-bw/allreduce_524288_busbw:0": 15.11,
        "nccl-bw/allreduce_524288_algbw:0": 8.64,
        "nccl-bw/allreduce_524288_time:0": 60.71,
        "nccl-bw/allreduce_1048576_busbw:0": 24.1,
        "nccl-bw/allreduce_1048576_algbw:0": 13.77,
        "nccl-bw/allreduce_1048576_time:0": 76.13,
        "nccl-bw/allreduce_2097152_busbw:0": 38.12,
        "nccl-bw/allreduce_2097152_algbw:0": 21.78,
        "nccl-bw/allreduce_2097152_time:0": 96.28,
        "nccl-bw/allreduce_4194304_busbw:0": 65.75,
        "nccl-bw/allreduce_4194304_algbw:0": 37.57,
        "nccl-bw/allreduce_4194304_time:0": 111.6,
        "nccl-bw/allreduce_8388608_busbw:0": 89.51,
        "nccl-bw/allreduce_8388608_algbw:0": 51.15,
        "nccl-bw/allreduce_8388608_time:0": 164.0,
        "nccl-bw/allreduce_16777216_busbw:0": 114.38,
        "nccl-bw/allreduce_16777216_algbw:0": 65.36,
        "nccl-bw/allreduce_16777216_time:0": 256.7,
        "nccl-bw/allreduce_33554432_busbw:0": 154.89,
        "nccl-bw/allreduce_33554432_algbw:0": 88.51,
        "nccl-bw/allreduce_33554432_time:0": 379.1,
        "nccl-bw/allreduce_67108864_busbw:0": 200.01,
        "nccl-bw/allreduce_67108864_algbw:0": 114.29,
        "nccl-bw/allreduce_67108864_time:0": 587.2,
        "nccl-bw/allreduce_134217728_busbw:0": 202.97,
        "nccl-bw/allreduce_134217728_algbw:0": 115.98,
        "nccl-bw/allreduce_134217728_time:0": 1157.2,
        "nccl-bw/allreduce_268435456_busbw:0": 221.82,
        "nccl-bw/allreduce_268435456_algbw:0": 126.75,
        "nccl-bw/allreduce_268435456_time:0": 2117.8,
        "nccl-bw/allreduce_536870912_busbw:0": 224.54,
        "nccl-bw/allreduce_536870912_algbw:0": 128.31,
        "nccl-bw/allreduce_536870912_time:0": 4184.2,
        "nccl-bw/allreduce_1073741824_busbw:0": 230.15,
        "nccl-bw/allreduce_1073741824_algbw:0": 131.51,
        "nccl-bw/allreduce_1073741824_time:0": 8164.5,
        "nccl-bw/allreduce_2147483648_busbw:0": 231.89,
        "nccl-bw/allreduce_2147483648_algbw:0": 132.51,
        "nccl-bw/allreduce_2147483648_time:0": 16207.0,
        "nccl-bw/allreduce_4294967296_busbw:0": 234.45,
        "nccl-bw/allreduce_4294967296_algbw:0": 133.97,
        "nccl-bw/allreduce_4294967296_time:0": 32059.0,
        "nccl-bw/allreduce_8589934592_busbw:0": 235.36,
        "nccl-bw/allreduce_8589934592_algbw:0": 134.49,
        "nccl-bw/allreduce_8589934592_time:0": 63870.0,
        "resnet_models/pytorch-resnet50/steptime_train_float32": 253.9552273229,
        "resnet_models/pytorch-resnet50/throughput_train_float32": 760.334809913,
        "resnet_models/pytorch-resnet50/steptime_train_float16": 200.0860618427,
        "resnet_models/pytorch-resnet50/throughput_train_float16": 971.0651430923,
        "resnet_models/pytorch-resnet101/steptime_train_float32": 389.0860509127,
        "resnet_models/pytorch-resnet101/throughput_train_float32": 496.117474093,
        "resnet_models/pytorch-resnet101/steptime_train_float16": 308.6274107918,
        "resnet_models/pytorch-resnet101/throughput_train_float16": 627.2056272195,
        "resnet_models/pytorch-resnet152/steptime_train_float32": 547.6558278315,
        "resnet_models/pytorch-resnet152/throughput_train_float32": 352.0709954335,
        "resnet_models/pytorch-resnet152/steptime_train_float16": 424.5809856802,
        "resnet_models/pytorch-resnet152/throughput_train_float16": 454.8335998154,
        "pytorch-sharding-matmul/allreduce": 10.574411869,
        "pytorch-sharding-matmul/allgather": 10.0846967697,
        "vgg_models/pytorch-vgg11/steptime_train_float32": 40.3528367169,
        "vgg_models/pytorch-vgg11/throughput_train_float32": 796.361593695,
        "vgg_models/pytorch-vgg11/steptime_train_float16": 24.1335148457,
        "vgg_models/pytorch-vgg11/throughput_train_float16": 1330.4113614585,
        "vgg_models/pytorch-vgg13/steptime_train_float32": 55.466310936,
        "vgg_models/pytorch-vgg13/throughput_train_float32": 580.2341074444,
        "vgg_models/pytorch-vgg13/steptime_train_float16": 33.3522899309,
        "vgg_models/pytorch-vgg13/throughput_train_float16": 962.5332023902,
        "vgg_models/pytorch-vgg16/steptime_train_float32": 65.225199447,
        "vgg_models/pytorch-vgg16/throughput_train_float32": 493.4268638876,
        "vgg_models/pytorch-vgg16/steptime_train_float16": 39.2528773518,
        "vgg_models/pytorch-vgg16/throughput_train_float16": 817.2008546148,
        "vgg_models/pytorch-vgg19/steptime_train_float32": 74.9348710524,
        "vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
        "vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
        "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377,
1403
1404
1405
1406
        "diagnosis/accept": false,
        "diagnosis/issue_num": 17,
        "diagnosis/category": "FailedTest",
        "diagnosis/issue_details": "mem-bw/D2H_Mem_BW:0_miss,mem-bw/D2H_Mem_BW:1_miss,mem-bw/D2H_Mem_BW:2_miss,mem-bw/D2H_Mem_BW:3_miss,mem-bw/D2H_Mem_BW:4_miss,mem-bw/D2H_Mem_BW:5_miss,mem-bw/D2H_Mem_BW:6_miss,mem-bw/D2H_Mem_BW:7_miss,mem-bw/H2D_Mem_BW:0_miss,mem-bw/H2D_Mem_BW:1_miss,mem-bw/H2D_Mem_BW:2_miss,mem-bw/H2D_Mem_BW:3_miss,mem-bw/H2D_Mem_BW:4_miss,mem-bw/H2D_Mem_BW:5_miss,mem-bw/H2D_Mem_BW:6_miss,mem-bw/H2D_Mem_BW:7_miss,mem-bw/return_code(VAL: 1.0000 Rule:lambda x:x>0)",
1407
1408
1409
        "Index": "sb-validation-03"
    }
]