rocm_bandwidth_test_report.cpp 11.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
// 
// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved.
// 
// Developed by:
// 
//                 AMD Research and AMD HSA Software Development
// 
//                 Advanced Micro Devices, Inc.
// 
//                 www.amd.com
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
// 
//  - Redistributions of source code must retain the above copyright notice,
//    this list of conditions and the following disclaimers.
//  - Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimers in
//    the documentation and/or other materials provided with the distribution.
//  - Neither the names of Advanced Micro Devices, Inc,
//    nor the names of its contributors may be used to endorse or promote
//    products derived from this Software without specific prior written
//    permission.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////

#include "common.hpp"
#include "rocm_bandwidth_test.hpp"

#include <iomanip>
#include <sstream>
#include <algorithm>

50
static void printRecord(size_t size, double avg_time,
51
                        double avg_bandwidth, double min_time,
52
53
54
                        double peak_bandwidth) {

  std::stringstream size_str;
55
56
57
  if (size < 1024) {
    size_str << size << " Bytes";
  } else if (size < 1024 * 1024) {
58
59
60
61
62
63
    size_str << size / 1024 << " KB";
  } else {
    size_str << size / (1024 * 1024) << " MB";
  }

  uint32_t format = 15;
64
  std::cout.precision(3);
65
66
67
68
69
70
  std::cout << std::fixed;
  std::cout.width(format);
  std::cout << size_str.str();
  std::cout.width(format);
  std::cout << (avg_time * 1e6);
  std::cout.width(format);
71
  std::cout << avg_bandwidth;
72
73
74
75
76
77
78
79
  std::cout.width(format);
  std::cout << (min_time * 1e6);
  std::cout.width(format);
  std::cout << peak_bandwidth;
  std::cout << std::endl;
}

static void printCopyBanner(uint32_t src_pool_id, uint32_t src_agent_type,
80
81
                            uint32_t dst_pool_id, uint32_t dst_agent_type,
                            bool unidir) {
82
83
84
85
86
87
88
89

  std::stringstream src_type;
  std::stringstream dst_type;
  (src_agent_type == 0) ? src_type <<  "Cpu" : src_type << "Gpu";
  (dst_agent_type == 0) ? dst_type <<  "Cpu" : dst_type << "Gpu";

  std::cout << std::endl;
  std::cout << "================";
90
91
92
93
94
95
  if (unidir) {
    std::cout << "    Unidirectional Benchmark Result";
  } else {
    std::cout << "    Bidirectional Benchmark Result";
  }
  std::cout << "    ================";
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
  std::cout << std::endl;
  std::cout << "================";
  std::cout << " Src Device Id: " << src_pool_id;
  std::cout << " Src Device Type: " << src_type.str();
  std::cout << " ================";
  std::cout << std::endl;
  std::cout << "================";
  std::cout << " Dst Device Id: " << dst_pool_id;
  std::cout << " Dst Device Type: " << dst_type.str();
  std::cout << " ================";
  std::cout << std::endl;
  std::cout << std::endl;

  uint32_t format = 15;
  std::cout.setf(ios::left);
  std::cout.width(format);
  std::cout << "Data Size";
  std::cout.width(format);
  std::cout << "Avg Time(us)";
  std::cout.width(format);
  std::cout << "Avg BW(GB/s)";
  std::cout.width(format);
  std::cout << "Min Time(us)";
  std::cout.width(format);
  std::cout << "Peak BW(GB/s)";
  std::cout << std::endl;
}

double RocmBandwidthTest::GetMinTime(std::vector<double>& vec) {

  std::sort(vec.begin(), vec.end());
  return vec.at(0);
}

double RocmBandwidthTest::GetMeanTime(std::vector<double>& vec) {

132
133
134
135
136
  // In validation mode we run only one iteration
  if (validate_) {
    return vec.at(0);
  }

137
  // Number of elements is ONE plus number of iterations
138
  std::sort(vec.begin(), vec.end());
139
  vec.erase(vec.end() - 1);
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159

  double mean = 0.0;
  int num = vec.size();
  for (int it = 0; it < num; it++) {
    mean += vec[it];
  }
  mean /= num;
  return mean;
}

void RocmBandwidthTest::Display() const {

  // Iterate through list of transactions and display its timing data
  uint32_t trans_size = trans_list_.size();
  if (trans_size == 0) {
    std::cout << std::endl;
    std::cout << "  Invalid Request" << std::endl;
    std::cout << std::endl;
    return;
  }
160
  
161
  if (validate_) {
162
    PrintVersion();
163
    DisplayDevInfo();
164
    PrintLinkPropsMatrix(LINK_PROP_ACCESS);
165
    DisplayValidationMatrix();
166
167
168
169
    return;
  }

  if (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR) {
170
    PrintVersion();
171
    DisplayDevInfo();
172
173
    PrintLinkPropsMatrix(LINK_PROP_ACCESS);
    PrintLinkPropsMatrix(LINK_PROP_WEIGHT);
174
175
176
177
    DisplayCopyTimeMatrix(true);
    return;
  }

178
179
  if (req_copy_all_bidir_ == REQ_COPY_ALL_BIDIR) {
    if (bw_default_run_ == NULL) {
180
      PrintVersion();
181
      DisplayDevInfo();
182
183
      PrintLinkPropsMatrix(LINK_PROP_ACCESS);
      PrintLinkPropsMatrix(LINK_PROP_WEIGHT);
184
185
186
187
188
    }
    DisplayCopyTimeMatrix(true);
    return;
  }

189
  if ((req_copy_bidir_ == REQ_COPY_BIDIR) ||
190
191
192
      (req_copy_unidir_ == REQ_COPY_UNIDIR) ||
      (req_concurrent_copy_bidir_ == REQ_CONCURRENT_COPY_BIDIR) ||
      (req_concurrent_copy_unidir_ == REQ_CONCURRENT_COPY_UNIDIR)) {
193
194
195
      PrintVersion();
  }

196
197
198
  for (uint32_t idx = 0; idx < trans_size; idx++) {
    async_trans_t trans = trans_list_[idx];
    if ((trans.req_type_ == REQ_COPY_BIDIR) ||
199
200
201
        (trans.req_type_ == REQ_COPY_UNIDIR) ||
        (trans.req_type_ == REQ_CONCURRENT_COPY_BIDIR) ||
        (trans.req_type_ == REQ_CONCURRENT_COPY_UNIDIR)) {
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
      DisplayCopyTime(trans);
    }
    if ((trans.req_type_ == REQ_READ) ||
        (trans.req_type_ == REQ_WRITE)) {
      DisplayIOTime(trans);
    }
  }
  std::cout << std::endl;
}

void RocmBandwidthTest::DisplayIOTime(async_trans_t& trans) const {

}

void RocmBandwidthTest::DisplayCopyTime(async_trans_t& trans) const {

  // Print Benchmark Header
  uint32_t src_idx = trans.copy.src_idx_;
  uint32_t dst_idx = trans.copy.dst_idx_;
  uint32_t src_dev_idx = pool_list_[src_idx].agent_index_;
  hsa_device_type_t src_dev_type = agent_list_[src_dev_idx].device_type_;
  uint32_t dst_dev_idx = pool_list_[dst_idx].agent_index_;
  hsa_device_type_t dst_dev_type = agent_list_[dst_dev_idx].device_type_;
225
226
227
228
  
  bool unidir = ((trans.req_type_ == REQ_COPY_UNIDIR) ||
                 (trans.req_type_ == REQ_CONCURRENT_COPY_UNIDIR));
  printCopyBanner(src_idx, src_dev_type, dst_idx, dst_dev_type, unidir);
229
230
231
232
233
234
235
236
237

  uint32_t size_len = size_list_.size();
  for (uint32_t idx = 0; idx < size_len; idx++) {
    printRecord(size_list_[idx], trans.avg_time_[idx],
                trans.avg_bandwidth_[idx], trans.min_time_[idx],
                trans.peak_bandwidth_[idx]);
  }
}

238
void RocmBandwidthTest::PopulatePerfMatrix(bool peak, double* perf_matrix) const {
239
240
241
242
243
244
245
246

  uint32_t trans_size = trans_list_.size();
  for (uint32_t idx = 0; idx < trans_size; idx++) {
    async_trans_t trans = trans_list_[idx];
    uint32_t src_idx = trans.copy.src_idx_;
    uint32_t dst_idx = trans.copy.dst_idx_;
    uint32_t src_dev_idx = pool_list_[src_idx].agent_index_;
    uint32_t dst_dev_idx = pool_list_[dst_idx].agent_index_;
247
248
249
250
251
252

    // For COPY_ALL_UNIDIR and COPY_ALL_BIDIR we use only one copy size
    double bandwidth = (peak) ? trans.peak_bandwidth_[0] : trans.avg_bandwidth_[0];
    perf_matrix[(src_dev_idx * agent_index_) + dst_dev_idx] = bandwidth;
    if (req_copy_all_bidir_ == REQ_COPY_ALL_BIDIR) {
      perf_matrix[(dst_dev_idx * agent_index_) + src_dev_idx] = bandwidth;
253
254
255
    }
  }

256
257
258
259
}

void RocmBandwidthTest::PrintPerfMatrix(bool validate, bool peak, double* perf_matrix) const {

260
261
262
263
264
265
266
  uint32_t format = 10;
  std::cout.setf(ios::left);

  std::cout.width(format);
  std::cout << "";
  std::cout.width(format);
  
267
268
269
270
271
272
273
274
275
276
  if (validate == false) { 
    if ((peak) && (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR)) {
      std::cout << "Unidirectional copy peak bandwidth GB/s";
    }
    
    if ((peak == false) && (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR)) {
      std::cout << "Unidirectional copy average bandwidth GB/s";
    }
    
    if ((peak) && (req_copy_all_bidir_ == REQ_COPY_ALL_BIDIR)) {
Bengt Lüers's avatar
Bengt Lüers committed
277
      std::cout << "Bidirectional copy peak bandwidth GB/s";
278
279
280
281
282
283
284
    }
    
    if ((peak == false) && (req_copy_all_bidir_ == REQ_COPY_ALL_BIDIR)) {
      std::cout << "Bidirectional copy average bandwidth GB/s";
    }
  } else {
    std::cout << "Data Path Validation";
285
286
287
288
  }

  std::cout << std::endl;
  std::cout << std::endl;
289
  std::cout.precision(3);
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
  std::cout << std::fixed;

  std::cout.width(format);
  std::cout << "";
  std::cout.width(format);
  std::cout << "D/D";
  format = 12;
  for (uint32_t idx0 = 0; idx0 < agent_index_; idx0++) {
    std::cout.width(format);
    std::stringstream agent_id;
    agent_id << idx0;
    std::cout << agent_id.str();
  }
  std::cout << std::endl;
  std::cout << std::endl;
  for (uint32_t idx0 = 0; idx0 < agent_index_; idx0++) {
    format = 10;
    std::cout.width(format);
    std::cout << "";
    std::stringstream agent_id;
    agent_id << idx0;
    std::cout.width(format);
    std::cout << agent_id.str();
    for (uint32_t idx1 = 0; idx1 < agent_index_; idx1++) {
      format = 12;
      std::cout.width(format);
      double value = perf_matrix[(idx0 * agent_index_) + idx1];
317
318
319
      if (validate) {
        if (value == 0) {
          std::cout << "N/A";
320
        } else if (value == VALIDATE_COPY_OP_FAILURE) {
321
322
323
324
          std::cout << "FAIL";
        } else {
          std::cout << "PASS";
        }
325
      } else {
326
327
328
329
330
        if (value == 0) {
          std::cout << "N/A";
        } else {
          std::cout << perf_matrix[(idx0 * agent_index_) + idx1];
        }
331
332
333
334
335
336
337
338
      }
    }
    std::cout << std::endl;
    std::cout << std::endl;
  }
  std::cout << std::endl;
}

339
void RocmBandwidthTest::DisplayCopyTimeMatrix(bool peak) const {
340
341

  double* perf_matrix = new double[agent_index_ * agent_index_]();
342
343
344
345
  PopulatePerfMatrix(peak, perf_matrix);
  PrintPerfMatrix(false, peak, perf_matrix);
  free(perf_matrix);
}
346

347
void RocmBandwidthTest::DisplayValidationMatrix() const {
348

349
350
351
352
  double* perf_matrix = new double[agent_index_ * agent_index_]();
  PopulatePerfMatrix(true, perf_matrix);
  PrintPerfMatrix(true, true, perf_matrix);
  free(perf_matrix);
353
354
}

355
356
357
358
359
360
361
362
363
364
365
366
void RocmBandwidthTest::DisplayDevInfo() const {

  uint32_t format = 10;
  std::cout.setf(ios::left);

  std::cout << std::endl;
  for (uint32_t idx = 0; idx < agent_index_; idx++) {
    uint32_t active = active_agents_list_[idx];
    if (active == 1) {
      std::cout.width(format);
      std::cout << "";
      std::cout << "Device: " << idx;
367
368
369
370
371
372
      std::cout << ",  " << agent_list_[idx].name_;
      bool gpuDevice = (agent_list_[idx].device_type_ == HSA_DEVICE_TYPE_GPU);
      if (gpuDevice) {
        std::cout << ",  " << agent_list_[idx].bdf_id_;
      }
      std::cout << std::endl;
373
374
375
376
377
378
    }
  }
  std::cout << std::endl;
}