EnvVars.hpp 31.6 KB
Newer Older
Gilbert Lee's avatar
Gilbert Lee committed
1
/*
gilbertlee-amd's avatar
gilbertlee-amd committed
2
Copyright (c) 2021-2023 Advanced Micro Devices, Inc. All rights reserved.
Gilbert Lee's avatar
Gilbert Lee committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#ifndef ENVVARS_HPP
#define ENVVARS_HPP

#include <algorithm>
27
28
#include <random>
#include <time.h>
29
#include "Compatibility.hpp"
gilbertlee-amd's avatar
gilbertlee-amd committed
30
31
#include "Kernels.hpp"

32
#define TB_VERSION "1.42"
Gilbert Lee's avatar
Gilbert Lee committed
33
34

extern char const MemTypeStr[];
gilbertlee-amd's avatar
gilbertlee-amd committed
35
extern char const ExeTypeStr[];
Gilbert Lee's avatar
Gilbert Lee committed
36

37
38
enum ConfigModeEnum
{
39
40
41
42
43
44
  CFG_FILE   = 0,
  CFG_P2P    = 1,
  CFG_SWEEP  = 2,
  CFG_SCALE  = 3,
  CFG_A2A    = 4,
  CFG_SCHMOO = 5
45
46
};

47
48
49
50
51
52
53
enum BlockOrderEnum
{
  ORDER_SEQUENTIAL  = 0,
  ORDER_INTERLEAVED = 1,
  ORDER_RANDOM      = 2
};

Gilbert Lee's avatar
Gilbert Lee committed
54
55
56
57
58
// This class manages environment variable that affect TransferBench
class EnvVars
{
public:
  // Default configuration values
59
  int const DEFAULT_NUM_WARMUPS          =  3;
Gilbert Lee's avatar
Gilbert Lee committed
60
61
  int const DEFAULT_NUM_ITERATIONS       = 10;
  int const DEFAULT_SAMPLING_FACTOR      =  1;
Gilbert Lee's avatar
Gilbert Lee committed
62

gilbertlee-amd's avatar
gilbertlee-amd committed
63
64
65
66
  // Peer-to-peer Benchmark preset defaults
  int const DEFAULT_P2P_NUM_CPU_SE    = 4;

  // Sweep-preset defaults
Gilbert Lee's avatar
Gilbert Lee committed
67
  std::string const DEFAULT_SWEEP_SRC = "CG";
gilbertlee-amd's avatar
gilbertlee-amd committed
68
  std::string const DEFAULT_SWEEP_EXE = "CDG";
Gilbert Lee's avatar
Gilbert Lee committed
69
70
71
72
73
74
  std::string const DEFAULT_SWEEP_DST = "CG";
  int const DEFAULT_SWEEP_MIN         = 1;
  int const DEFAULT_SWEEP_MAX         = 24;
  int const DEFAULT_SWEEP_TEST_LIMIT  = 0;
  int const DEFAULT_SWEEP_TIME_LIMIT  = 0;

Gilbert Lee's avatar
Gilbert Lee committed
75
  // Environment variables
76
  int alwaysValidate;    // Validate after each iteration instead of once after all iterations
77
  int blockSize;         // Size of each threadblock (must be multiple of 64)
Gilbert Lee's avatar
Gilbert Lee committed
78
  int blockBytes;        // Each CU, except the last, gets a multiple of this many bytes to copy
79
  int blockOrder;        // How blocks are ordered in single-stream mode (0=Sequential, 1=Interleaved, 2=Random)
Gilbert Lee's avatar
Gilbert Lee committed
80
  int byteOffset;        // Byte-offset for memory allocations
81
  int continueOnError;   // Continue tests even after mismatch detected
82
  int hideEnv;           // Skip printing environment variable
Gilbert Lee's avatar
Gilbert Lee committed
83
84
  int numCpuDevices;     // Number of CPU devices to use (defaults to # NUMA nodes detected)
  int numGpuDevices;     // Number of GPU devices to use (defaults to # HIP devices detected)
Gilbert Lee's avatar
Gilbert Lee committed
85
86
87
88
89
  int numIterations;     // Number of timed iterations to perform.  If negative, run for -numIterations seconds instead
  int numWarmups;        // Number of un-timed warmup iterations to perform
  int outputToCsv;       // Output in CSV format
  int samplingFactor;    // Affects how many different values of N are generated (when N set to 0)
  int sharedMemBytes;    // Amount of shared memory to use per threadblock
90
  int showIterations;    // Show per-iteration timing info
Gilbert Lee's avatar
Gilbert Lee committed
91
92
  int useInteractive;    // Pause for user-input before starting transfer loop
  int usePcieIndexing;   // Base GPU indexing on PCIe address instead of HIP device
93
  int usePrepSrcKernel;  // Use GPU kernel to prepare source data instead of copy (can't be used with fillPattern)
gilbertlee-amd's avatar
gilbertlee-amd committed
94
  int useSingleStream;   // Use a single stream per GPU GFX executor instead of stream per Transfer
95
  int useXccFilter;      // Use XCC filtering (experimental)
96
  int validateDirect;    // Validate GPU destination memory directly instead of staging GPU memory on host
Gilbert Lee's avatar
Gilbert Lee committed
97
98

  std::vector<float> fillPattern; // Pattern of floats used to fill source data
99
  std::vector<uint32_t> cuMask;   // Bit-vector representing the CU mask
100
  std::vector<std::vector<int>> prefXccTable;
Gilbert Lee's avatar
Gilbert Lee committed
101

102
  // Environment variables only for P2P preset
gilbertlee-amd's avatar
gilbertlee-amd committed
103
  int numCpuSubExecs;    // Number of CPU subexecttors to use
104
105
106
107
  int numGpuSubExecs;    // Number of GPU subexecutors to use
  int p2pMode;           // Both = 0, Unidirectional = 1, Bidirectional = 2
  int useDmaCopy;        // Use DMA copy instead of GPU copy
  int useRemoteRead;     // Use destination memory type as executor instead of source memory type
108
  int useFineGrain;      // Use fine-grained memory
gilbertlee-amd's avatar
gilbertlee-amd committed
109

Gilbert Lee's avatar
Gilbert Lee committed
110
111
112
113
114
  // Environment variables only for Sweep-preset
  int sweepMin;          // Min number of simultaneous Transfers to be executed per test
  int sweepMax;          // Max number of simulatneous Transfers to be executed per test
  int sweepTestLimit;    // Max number of tests to run during sweep (0 = no limit)
  int sweepTimeLimit;    // Max number of seconds to run sweep for  (0 = no limit)
115
116
117
118
  int sweepXgmiMin;      // Min number of XGMI hops for Transfers
  int sweepXgmiMax;      // Max number of XGMI hops for Transfers (-1 = no limit)
  int sweepSeed;         // Random seed to use
  int sweepRandBytes;    // Whether or not to use random number of bytes per Transfer
Gilbert Lee's avatar
Gilbert Lee committed
119
120
121
122
  std::string sweepSrc;  // Set of src memory types to be swept
  std::string sweepExe;  // Set of executors to be swept
  std::string sweepDst;  // Set of dst memory types to be swept

123
124
125
  // Enviroment variables only for A2A preset
  int a2aDirect;         // Only execute on links that are directly connected

gilbertlee-amd's avatar
gilbertlee-amd committed
126
127
128
129
  // Developer features
  int enableDebug;       // Enable debug output
  int gpuKernel;         // Which GPU kernel to use

130
131
132
133
134
135
  // Used to track current configuration mode
  ConfigModeEnum configMode;

  // Random generator
  std::default_random_engine *generator;

136
137
138
  // Track how many CPUs are available per NUMA node
  std::vector<int> numCpusPerNuma;

139
140
  std::vector<int> wallClockPerDeviceMhz;

141
142
  std::vector<std::set<int>> xccIdsPerDevice;

Gilbert Lee's avatar
Gilbert Lee committed
143
144
145
146
  // Constructor that collects values
  EnvVars()
  {
    int maxSharedMemBytes = 0;
gilbertlee-amd's avatar
gilbertlee-amd committed
147
148
    HIP_CALL(hipDeviceGetAttribute(&maxSharedMemBytes,
                                   hipDeviceAttributeMaxSharedMemoryPerMultiprocessor, 0));
149
150
151
152
153
154
#if !defined(__NVCC__)
    int defaultSharedMemBytes = maxSharedMemBytes / 2 + 1;
#else
    int defaultSharedMemBytes = 0;
#endif

gilbertlee-amd's avatar
gilbertlee-amd committed
155
156
    int numDeviceCUs = 0;
    HIP_CALL(hipDeviceGetAttribute(&numDeviceCUs, hipDeviceAttributeMultiprocessorCount, 0));
Gilbert Lee's avatar
Gilbert Lee committed
157

Gilbert Lee's avatar
Gilbert Lee committed
158
159
    int numDetectedCpus = numa_num_configured_nodes();
    int numDetectedGpus;
gilbertlee-amd's avatar
gilbertlee-amd committed
160
161
162
163
164
165
166
167
168
169
170
171
    HIP_CALL(hipGetDeviceCount(&numDetectedGpus));

    hipDeviceProp_t prop;
    HIP_CALL(hipGetDeviceProperties(&prop, 0));
    std::string fullName = prop.gcnArchName;
    std::string archName = fullName.substr(0, fullName.find(':'));

    // Different hardware pick different GPU kernels
    // This performance difference is generally only noticable when executing fewer CUs
    int defaultGpuKernel = 0;
    if      (archName == "gfx906") defaultGpuKernel = 13;
    else if (archName == "gfx90a") defaultGpuKernel = 9;
172
173
    else if (archName == "gfx940") defaultGpuKernel = 6;
    else if (archName == "gfx941") defaultGpuKernel = 6;
174
    else if (archName == "gfx942") defaultGpuKernel = 3;
Gilbert Lee's avatar
Gilbert Lee committed
175

176
    alwaysValidate    = GetEnvVar("ALWAYS_VALIDATE"     , 0);
177
    blockSize         = GetEnvVar("BLOCK_SIZE"          , 256);
Gilbert Lee's avatar
Gilbert Lee committed
178
    blockBytes        = GetEnvVar("BLOCK_BYTES"         , 256);
179
    blockOrder        = GetEnvVar("BLOCK_ORDER"         , 0);
Gilbert Lee's avatar
Gilbert Lee committed
180
    byteOffset        = GetEnvVar("BYTE_OFFSET"         , 0);
181
    continueOnError   = GetEnvVar("CONTINUE_ON_ERROR"   , 0);
182
    hideEnv           = GetEnvVar("HIDE_ENV"            , 0);
Gilbert Lee's avatar
Gilbert Lee committed
183
184
    numCpuDevices     = GetEnvVar("NUM_CPU_DEVICES"     , numDetectedCpus);
    numGpuDevices     = GetEnvVar("NUM_GPU_DEVICES"     , numDetectedGpus);
Gilbert Lee's avatar
Gilbert Lee committed
185
186
187
188
    numIterations     = GetEnvVar("NUM_ITERATIONS"      , DEFAULT_NUM_ITERATIONS);
    numWarmups        = GetEnvVar("NUM_WARMUPS"         , DEFAULT_NUM_WARMUPS);
    outputToCsv       = GetEnvVar("OUTPUT_TO_CSV"       , 0);
    samplingFactor    = GetEnvVar("SAMPLING_FACTOR"     , DEFAULT_SAMPLING_FACTOR);
189
    sharedMemBytes    = GetEnvVar("SHARED_MEM_BYTES"    , defaultSharedMemBytes);
190
    showIterations    = GetEnvVar("SHOW_ITERATIONS"     , 0);
Gilbert Lee's avatar
Gilbert Lee committed
191
192
    useInteractive    = GetEnvVar("USE_INTERACTIVE"     , 0);
    usePcieIndexing   = GetEnvVar("USE_PCIE_INDEX"      , 0);
193
    usePrepSrcKernel  = GetEnvVar("USE_PREP_KERNEL"     , 0);
194
    useSingleStream   = GetEnvVar("USE_SINGLE_STREAM"   , 1);
195
    useXccFilter      = GetEnvVar("USE_XCC_FILTER"      , 0);
196
    validateDirect    = GetEnvVar("VALIDATE_DIRECT"     , 0);
gilbertlee-amd's avatar
gilbertlee-amd committed
197
198
    enableDebug       = GetEnvVar("DEBUG"               , 0);
    gpuKernel         = GetEnvVar("GPU_KERNEL"          , defaultGpuKernel);
Gilbert Lee's avatar
Gilbert Lee committed
199

gilbertlee-amd's avatar
gilbertlee-amd committed
200
    // P2P Benchmark related
201
202
    useDmaCopy        = GetEnvVar("USE_GPU_DMA"         , 0); // Needed for numGpuSubExec

203
    numCpuSubExecs    = GetEnvVar("NUM_CPU_SE"          , DEFAULT_P2P_NUM_CPU_SE);
204
    numGpuSubExecs    = GetEnvVar("NUM_GPU_SE"          , useDmaCopy ? 1 : numDeviceCUs);
205
    p2pMode           = GetEnvVar("P2P_MODE"            , 0);
206
207
    useRemoteRead     = GetEnvVar("USE_REMOTE_READ"     , 0);
    useFineGrain      = GetEnvVar("USE_FINE_GRAIN"      , 0);
gilbertlee-amd's avatar
gilbertlee-amd committed
208
209

    // Sweep related
210
211
212
213
214
215
216
217
218
219
220
    sweepMin          = GetEnvVar("SWEEP_MIN"           , DEFAULT_SWEEP_MIN);
    sweepMax          = GetEnvVar("SWEEP_MAX"           , DEFAULT_SWEEP_MAX);
    sweepSrc          = GetEnvVar("SWEEP_SRC"           , DEFAULT_SWEEP_SRC);
    sweepExe          = GetEnvVar("SWEEP_EXE"           , DEFAULT_SWEEP_EXE);
    sweepDst          = GetEnvVar("SWEEP_DST"           , DEFAULT_SWEEP_DST);
    sweepTestLimit    = GetEnvVar("SWEEP_TEST_LIMIT"    , DEFAULT_SWEEP_TEST_LIMIT);
    sweepTimeLimit    = GetEnvVar("SWEEP_TIME_LIMIT"    , DEFAULT_SWEEP_TIME_LIMIT);
    sweepXgmiMin      = GetEnvVar("SWEEP_XGMI_MIN"      , 0);
    sweepXgmiMax      = GetEnvVar("SWEEP_XGMI_MAX"      , -1);
    sweepRandBytes    = GetEnvVar("SWEEP_RAND_BYTES"    , 0);

221
222
223
    // A2A Benchmark related
    a2aDirect         = GetEnvVar("A2A_DIRECT"          , 1);

224
225
226
227
    // Determine random seed
    char *sweepSeedStr = getenv("SWEEP_SEED");
    sweepSeed = (sweepSeedStr != NULL ? atoi(sweepSeedStr) : time(NULL));
    generator = new std::default_random_engine(sweepSeed);
Gilbert Lee's avatar
Gilbert Lee committed
228

Gilbert Lee's avatar
Gilbert Lee committed
229
230
231
232
    // Check for fill pattern
    char* pattern = getenv("FILL_PATTERN");
    if (pattern != NULL)
    {
233
234
235
236
237
238
      if (usePrepSrcKernel)
      {
        printf("[ERROR] Unable to use FILL_PATTERN and USE_PREP_KERNEL together\n");
        exit(1);
      }

Gilbert Lee's avatar
Gilbert Lee committed
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
      int patternLen = strlen(pattern);
      if (patternLen % 2)
      {
        printf("[ERROR] FILL_PATTERN must contain an even-number of hex digits\n");
        exit(1);
      }

      // Read in bytes
      std::vector<unsigned char> bytes;
      unsigned char val = 0;
      for (int i = 0; i < patternLen; i++)
      {
        if ('0' <= pattern[i] && pattern[i] <= '9')
          val += (pattern[i] - '0');
        else if ('A' <= pattern[i] && pattern[i] <= 'F')
          val += (pattern[i] - 'A' + 10);
        else if ('a' <= pattern[i] && pattern[i] <= 'f')
          val += (pattern[i] - 'a' + 10);
        else
        {
          printf("[ERROR] FILL_PATTERN must contain an even-number of hex digits (0-9'/a-f/A-F).  (not %c)\n", pattern[i]);
          exit(1);
        }

        if (i % 2 == 0)
          val <<= 4;
        else
        {
          bytes.push_back(val);
          val = 0;
        }
      }

      // Reverse bytes (input is assumed to be given in big-endian)
      std::reverse(bytes.begin(), bytes.end());

      // Figure out how many copies of the pattern are necessary to fill a 4-byte float properly
      int copies;
      switch (patternLen % 8)
      {
      case 0:  copies = 1; break;
      case 4:  copies = 2; break;
      default: copies = 4; break;
      }

      // Fill floats
      int numFloats = copies * patternLen / 8;
      fillPattern.resize(numFloats);
      unsigned char* rawData = (unsigned char*) fillPattern.data();
      for (int i = 0; i < numFloats * 4; i++)
        rawData[i] = bytes[i % bytes.size()];
    }
    else fillPattern.clear();

293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
    // Check for CU mask
    cuMask.clear();
    char* cuMaskStr = getenv("CU_MASK");
    if (cuMaskStr != NULL)
    {
#if defined(__NVCC__)
      printf("[WARN] CU_MASK is not supported in CUDA\n");
#else
      std::vector<std::pair<int, int>> ranges;
      int maxCU = 0;
      char* token = strtok(cuMaskStr, ",");
      while (token)
      {
        int start, end;
        if (sscanf(token, "%d-%d", &start, &end) == 2)
        {
          ranges.push_back(std::make_pair(std::min(start, end), std::max(start, end)));
          maxCU = std::max(maxCU, std::max(start, end));
        }
        else if (sscanf(token, "%d", &start) == 1)
        {
          ranges.push_back(std::make_pair(start, start));
          maxCU = std::max(maxCU, start);
        }
        else
        {
          printf("[ERROR] Unrecognized token [%s]\n", token);
          exit(1);
        }
        token = strtok(NULL, ",");
      }
      cuMask.resize(maxCU / 32 + 1, 0);

      for (auto range : ranges)
      {
        for (int i = range.first; i <= range.second; i++)
        {
          cuMask[i / 32] |= (1 << (i % 32));
        }
      }
#endif
    }

336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
    // Figure out number of xccs per device
    int maxNumXccs = 64;
    xccIdsPerDevice.resize(numGpuDevices);
    for (int i = 0; i < numGpuDevices; i++)
    {
      int* data;
      HIP_CALL(hipSetDevice(i));
      HIP_CALL(hipHostMalloc((void**)&data, maxNumXccs * sizeof(int)));
      CollectXccIdsKernel<<<maxNumXccs, 1>>>(data);
      HIP_CALL(hipDeviceSynchronize());

      xccIdsPerDevice[i].clear();
      for (int j = 0; j < maxNumXccs; j++)
        xccIdsPerDevice[i].insert(data[j]);

      HIP_CALL(hipHostFree(data));
    }

    // Parse preferred XCC table (if provided
355
356
357
    prefXccTable.resize(numGpuDevices);
    for (int i = 0; i < numGpuDevices; i++)
    {
358
      prefXccTable[i].resize(numGpuDevices, -1);
359
360
361
362
    }

    char* prefXccStr = getenv("XCC_PREF_TABLE");
    if (prefXccStr)
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
    {
      char* token = strtok(prefXccStr, ",");
      int tokenCount = 0;
      while (token)
      {
        int xccId;
        if (sscanf(token, "%d", &xccId) == 1)
        {
          int src = tokenCount / numGpuDevices;
          int dst = tokenCount % numGpuDevices;
          if (xccIdsPerDevice[src].count(xccId) == 0)
          {
            printf("[ERROR] GPU %d does not contain XCC %d\n", src, xccId);
            exit(1);
          }
          prefXccTable[src][dst] = xccId;

          tokenCount++;
          if (tokenCount == (numGpuDevices * numGpuDevices)) break;
        }
        else
        {
          printf("[ERROR] Unrecognized token [%s]\n", token);
          exit(1);
        }
        token = strtok(NULL, ",");
      }
    }

Gilbert Lee's avatar
Gilbert Lee committed
392
    // Perform some basic validation
Gilbert Lee's avatar
Gilbert Lee committed
393
394
395
396
397
398
399
400
401
402
    if (numCpuDevices > numDetectedCpus)
    {
      printf("[ERROR] Number of CPUs to use (%d) cannot exceed number of detected CPUs (%d)\n", numCpuDevices, numDetectedCpus);
      exit(1);
    }
    if (numGpuDevices > numDetectedGpus)
    {
      printf("[ERROR] Number of GPUs to use (%d) cannot exceed number of detected GPUs (%d)\n", numGpuDevices, numDetectedGpus);
      exit(1);
    }
403
404
405
406
407
408
409
410
411
412
    if (blockSize % 64)
    {
      printf("[ERROR] BLOCK_SIZE (%d) must be a multiple of 64\n", blockSize);
      exit(1);
    }
    if (blockSize > MAX_BLOCKSIZE)
    {
      printf("[ERROR] BLOCK_SIZE (%d) must be less than %d\n", blockSize, MAX_BLOCKSIZE);
      exit(1);
    }
Gilbert Lee's avatar
Gilbert Lee committed
413
414
415
416
417
    if (byteOffset % sizeof(float))
    {
      printf("[ERROR] BYTE_OFFSET must be set to multiple of %lu\n", sizeof(float));
      exit(1);
    }
418
419
420
421
422
    if (blockOrder < 0 || blockOrder > 2)
    {
      printf("[ERROR] BLOCK_ORDER must be 0 (Sequential), 1 (Interleaved), or 2 (Random)\n");
      exit(1);
    }
Gilbert Lee's avatar
Gilbert Lee committed
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
    if (numWarmups < 0)
    {
      printf("[ERROR] NUM_WARMUPS must be set to a non-negative number\n");
      exit(1);
    }
    if (samplingFactor < 1)
    {
      printf("[ERROR] SAMPLING_FACTOR must be greater or equal to 1\n");
      exit(1);
    }
    if (sharedMemBytes < 0 || sharedMemBytes > maxSharedMemBytes)
    {
      printf("[ERROR] SHARED_MEM_BYTES must be between 0 and %d\n", maxSharedMemBytes);
      exit(1);
    }
    if (blockBytes <= 0 || blockBytes % 4)
    {
      printf("[ERROR] BLOCK_BYTES must be a positive multiple of 4\n");
      exit(1);
    }
gilbertlee-amd's avatar
gilbertlee-amd committed
443
444
445
446
447
448
449
    if (numGpuSubExecs <= 0)
    {
      printf("[ERROR] NUM_GPU_SE must be greater than 0\n");
      exit(1);
    }

    if (numCpuSubExecs <= 0)
Gilbert Lee's avatar
Gilbert Lee committed
450
    {
gilbertlee-amd's avatar
gilbertlee-amd committed
451
      printf("[ERROR] NUM_CPU_SE must be greater than 0\n");
Gilbert Lee's avatar
Gilbert Lee committed
452
453
      exit(1);
    }
Gilbert Lee's avatar
Gilbert Lee committed
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484

    for (auto ch : sweepSrc)
    {
      if (!strchr(MemTypeStr, ch))
      {
        printf("[ERROR] Unrecognized memory type '%c' specified for sweep source\n", ch);
        exit(1);
      }
      if (strchr(sweepSrc.c_str(), ch) != strrchr(sweepSrc.c_str(), ch))
      {
        printf("[ERROR] Duplicate memory type '%c' specified for sweep source\n", ch);
        exit(1);
      }
    }

    for (auto ch : sweepDst)
    {
      if (!strchr(MemTypeStr, ch))
      {
        printf("[ERROR] Unrecognized memory type '%c' specified for sweep destination\n", ch);
        exit(1);
      }
      if (strchr(sweepDst.c_str(), ch) != strrchr(sweepDst.c_str(), ch))
      {
        printf("[ERROR] Duplicate memory type '%c' specified for sweep destination\n", ch);
        exit(1);
      }
    }

    for (auto ch : sweepExe)
    {
gilbertlee-amd's avatar
gilbertlee-amd committed
485
      if (!strchr(ExeTypeStr, ch))
Gilbert Lee's avatar
Gilbert Lee committed
486
487
488
489
490
491
492
493
494
495
      {
        printf("[ERROR] Unrecognized executor type '%c' specified for sweep executor\n", ch);
        exit(1);
      }
      if (strchr(sweepExe.c_str(), ch) != strrchr(sweepExe.c_str(), ch))
      {
        printf("[ERROR] Duplicate executor type '%c' specified for sweep executor\n", ch);
        exit(1);
      }
    }
gilbertlee-amd's avatar
gilbertlee-amd committed
496
497
498
499
500
    if (gpuKernel < 0 || gpuKernel > NUM_GPU_KERNELS)
    {
      printf("[ERROR] GPU kernel must be between 0 and %d\n", NUM_GPU_KERNELS);
      exit(1);
    }
501
502
503
504
505
506

    // Determine how many CPUs exit per NUMA node (to avoid executing on NUMA without CPUs)
    numCpusPerNuma.resize(numDetectedCpus);
    int const totalCpus = numa_num_configured_cpus();
    for (int i = 0; i < totalCpus; i++)
      numCpusPerNuma[numa_node_of_cpu(i)]++;
gilbertlee-amd's avatar
gilbertlee-amd committed
507

508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
    // Build array of wall clock rates per GPU device
    wallClockPerDeviceMhz.resize(numDetectedGpus);
    for (int i = 0; i < numDetectedGpus; i++)
    {
#if defined(__NVCC__)
      // NOTE: wallClock doesn't exist in CUDA.  This may need to be adjusted / run with fixed clocks
      wallClockPerDeviceMhz[i] = 1410000;
#else
      hipDeviceProp_t prop;
      HIP_CALL(hipGetDeviceProperties(&prop, i));
      int value = 25000;
      std::string fullName = prop.gcnArchName;
      std::string archName = fullName.substr(0, fullName.find(':'));
      if (archName == "gfx940" || archName == "gfx941" || archName == "gfx942")
        wallClockPerDeviceMhz[i] = 100000;
      else
        wallClockPerDeviceMhz[i] = 25000;
#endif
    }

gilbertlee-amd's avatar
gilbertlee-amd committed
528
529
530
531
532
533
534
535
536
537
538
539
    // Check for deprecated env vars
    if (getenv("USE_HIP_CALL"))
    {
      printf("[WARN] USE_HIP_CALL has been deprecated.  Please use DMA executor 'D' or set USE_GPU_DMA for P2P-Benchmark preset\n");
      exit(1);
    }

    char* enableSdma = getenv("HSA_ENABLE_SDMA");
    if (enableSdma && !strcmp(enableSdma, "0"))
    {
      printf("[WARN] DMA functionality disabled due to environment variable HSA_ENABLE_SDMA=0.  Copies will fallback to blit kernels\n");
    }
Gilbert Lee's avatar
Gilbert Lee committed
540
541
542
543
544
545
546
  }

  // Display info on the env vars that can be used
  static void DisplayUsage()
  {
    printf("Environment variables:\n");
    printf("======================\n");
547
    printf(" ALWAYS_VALIDATE        - Validate after each iteration instead of once after all iterations\n");
548
549
550
    printf(" BLOCK_SIZE             - # of threads per threadblock (Must be multiple of 64). Defaults to 256\n");
    printf(" BLOCK_BYTES            - Each CU (except the last) receives a multiple of BLOCK_BYTES to copy\n");
    printf(" BLOCK_ORDER            - Threadblock ordering in single-stream mode (0=Serial, 1=Interleaved, 2=Random)\n");
Gilbert Lee's avatar
Gilbert Lee committed
551
    printf(" BYTE_OFFSET            - Initial byte-offset for memory allocations.  Must be multiple of 4. Defaults to 0\n");
552
    printf(" CONTINUE_ON_ERROR      - Continue tests even after mismatch detected\n");
553
    printf(" CU_MASK                - CU mask for streams specified in hex digits (0-0,a-f,A-F)\n");
Gilbert Lee's avatar
Gilbert Lee committed
554
    printf(" FILL_PATTERN=STR       - Fill input buffer with pattern specified in hex digits (0-9,a-f,A-F).  Must be even number of digits, (byte-level big-endian)\n");
555
    printf(" HIDE_ENV               - Hide environment variable value listing\n");
Gilbert Lee's avatar
Gilbert Lee committed
556
    printf(" NUM_CPU_DEVICES=X      - Restrict number of CPUs to X.  May not be greater than # detected NUMA nodes\n");
gilbertlee-amd's avatar
gilbertlee-amd committed
557
    printf(" NUM_GPU_DEVICES=X      - Restrict number of GPUs to X.  May not be greater than # detected HIP devices\n");
Gilbert Lee's avatar
Gilbert Lee committed
558
559
560
561
562
    printf(" NUM_ITERATIONS=I       - Perform I timed iteration(s) per test\n");
    printf(" NUM_WARMUPS=W          - Perform W untimed warmup iteration(s) per test\n");
    printf(" OUTPUT_TO_CSV          - Outputs to CSV format if set\n");
    printf(" SAMPLING_FACTOR=F      - Add F samples (when possible) between powers of 2 when auto-generating data sizes\n");
    printf(" SHARED_MEM_BYTES=X     - Use X shared mem bytes per threadblock, potentially to avoid multiple threadblocks per CU\n");
563
    printf(" SHOW_ITERATIONS        - Show per-iteration timing info\n");
Gilbert Lee's avatar
Gilbert Lee committed
564
565
    printf(" USE_INTERACTIVE        - Pause for user-input before starting transfer loop\n");
    printf(" USE_PCIE_INDEX         - Index GPUs by PCIe address-ordering instead of HIP-provided indexing\n");
566
    printf(" USE_PREP_KERNEL        - Use GPU kernel to initialize source data array pattern\n");
gilbertlee-amd's avatar
gilbertlee-amd committed
567
    printf(" USE_SINGLE_STREAM      - Use a single stream per GPU GFX executor instead of stream per Transfer\n");
568
    printf(" USE_XCC_FILTER         - Use XCC filtering (experimental)\n");
569
    printf(" VALIDATE_DIRECT        - Validate GPU destination memory directly instead of staging GPU memory on host\n");
Gilbert Lee's avatar
Gilbert Lee committed
570
571
  }

572
573
574
575
576
577
578
  // Helper macro to switch between CSV and terminal output
#define PRINT_EV(NAME, VALUE, DESCRIPTION)                              \
  printf("%-20s%s%12d%s%s\n", NAME, outputToCsv ? "," : " = ", VALUE, outputToCsv ? "," : " : ",  (DESCRIPTION).c_str())

#define PRINT_ES(NAME, VALUE, DESCRIPTION)                           \
  printf("%-20s%s%12s%s%s\n", NAME, outputToCsv ? "," : " = ", VALUE, outputToCsv ? "," : " : ",  (DESCRIPTION).c_str())

Gilbert Lee's avatar
Gilbert Lee committed
579
580
581
582
583
  // Display env var settings
  void DisplayEnvVars() const
  {
    if (!outputToCsv)
    {
584
      printf("TransferBench v%s\n", TB_VERSION);
585
      printf("===============================================================\n");
586
      if (!hideEnv) printf("[Common]                              (Suppress by setting HIDE_ENV=1)\n");
Gilbert Lee's avatar
Gilbert Lee committed
587
    }
588
    else if (!hideEnv)
589
      printf("EnvVar,Value,Description,(TransferBench v%s)\n", TB_VERSION);
590
    if (hideEnv) return;
591
592
    PRINT_EV("ALWAYS_VALIDATE", alwaysValidate,
             std::string("Validating after ") + (alwaysValidate ? "each iteration" : "all iterations"));
593
594
    PRINT_EV("BLOCK_SIZE", blockSize,
             std::string("Threadblock size of " + std::to_string(blockSize)));
595
596
    PRINT_EV("BLOCK_BYTES", blockBytes,
             std::string("Each CU gets a multiple of " + std::to_string(blockBytes) + " bytes to copy"));
597
598
599
600
    PRINT_EV("BLOCK_ORDER", blockOrder,
             std::string("Transfer blocks order: " + std::string((blockOrder == 0 ? "Sequential"  :
                                                                  blockOrder == 1 ? "Interleaved" :
                                                                                    "Random"))));
601
602
603
604
    PRINT_EV("BYTE_OFFSET", byteOffset,
             std::string("Using byte offset of " + std::to_string(byteOffset)));
    PRINT_EV("CONTINUE_ON_ERROR", continueOnError,
             std::string(continueOnError ? "Continue on mismatch error" : "Stop after first error"));
605
606
    PRINT_EV("CU_MASK", getenv("CU_MASK") ? 1 : 0,
             (cuMask.size() ? GetCuMaskDesc() : "All"));
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
    PRINT_EV("FILL_PATTERN", getenv("FILL_PATTERN") ? 1 : 0,
             (fillPattern.size() ? std::string(getenv("FILL_PATTERN")) : PrepSrcValueString()));
    PRINT_EV("GPU_KERNEL", gpuKernel,
             std::string("Using GPU kernel ") + std::to_string(gpuKernel) + " [" + std::string(GpuKernelNames[gpuKernel]) + "]");
    PRINT_EV("NUM_CPU_DEVICES", numCpuDevices,
             std::string("Using ") + std::to_string(numCpuDevices) + " CPU devices");
    PRINT_EV("NUM_GPU_DEVICES", numGpuDevices,
             std::string("Using ") + std::to_string(numGpuDevices) + " GPU devices");
    PRINT_EV("NUM_ITERATIONS", numIterations,
             std::string("Running ") + std::to_string(numIterations > 0 ? numIterations : -numIterations) + " "
             + (numIterations > 0 ? " timed iteration(s)" : "seconds(s) per Test"));
    PRINT_EV("NUM_WARMUPS", numWarmups,
             std::string("Running " + std::to_string(numWarmups) + " warmup iteration(s) per Test"));
    PRINT_EV("SHARED_MEM_BYTES", sharedMemBytes,
             std::string("Using " + std::to_string(sharedMemBytes) + " shared mem per threadblock"));
622
623
    PRINT_EV("SHOW_ITERATIONS", showIterations,
             std::string(showIterations ? "Showing" : "Hiding") + " per-iteration timing");
624
625
626
627
628
629
630
631
    PRINT_EV("USE_INTERACTIVE", useInteractive,
             std::string("Running in ") + (useInteractive ? "interactive" : "non-interactive") + " mode");
    PRINT_EV("USE_PCIE_INDEX", usePcieIndexing,
             std::string("Use ") + (usePcieIndexing ? "PCIe" : "HIP") + " GPU device indexing");
    PRINT_EV("USE_PREP_KERNEL", usePrepSrcKernel,
             std::string("Using ") + (usePrepSrcKernel ? "GPU kernels" : "hipMemcpy") + " to initialize source data");
    PRINT_EV("USE_SINGLE_STREAM", useSingleStream,
             std::string("Using single stream per ") + (useSingleStream ? "device" : "Transfer"));
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
    PRINT_EV("USE_XCC_FILTER", useXccFilter,
             std::string("XCC filtering ") + (useXccFilter ? "enabled" : "disabled"));
    if (useXccFilter)
    {
      printf("%36s: Preferred XCC Table (XCC_PREF_TABLE)\n", "");
      printf("%36s:         ", "");
      for (int i = 0; i < numGpuDevices; i++) printf(" %3d", i); printf(" (#XCCs)\n");
      for (int i = 0; i < numGpuDevices; i++)
      {
        printf("%36s: GPU %3d ", "", i);
        for (int j = 0; j < numGpuDevices; j++)
          printf(" %3d", prefXccTable[i][j]);
        printf(" %3lu\n", xccIdsPerDevice[i].size());
      }
    }
647
648
649
    PRINT_EV("VALIDATE_DIRECT", validateDirect,
             std::string("Validate GPU destination memory ") + (validateDirect ? "directly" : "via CPU staging buffer"));
    printf("\n");
650
651
652

    if (blockOrder != ORDER_SEQUENTIAL && !useSingleStream)
      printf("[WARN] BLOCK_ORDER is ignored if USE_SINGLE_STREAM is not enabled\n");
Gilbert Lee's avatar
Gilbert Lee committed
653
654
  };

gilbertlee-amd's avatar
gilbertlee-amd committed
655
656
657
  // Display env var for P2P Benchmark preset
  void DisplayP2PBenchmarkEnvVars() const
  {
658
    DisplayEnvVars();
659

660
    if (hideEnv) return;
661

gilbertlee-amd's avatar
gilbertlee-amd committed
662
    if (!outputToCsv)
663
664
665
666
667
668
      printf("[P2P Related]\n");

    PRINT_EV("NUM_CPU_SE", numCpuSubExecs,
             std::string("Using ") + std::to_string(numCpuSubExecs) + " CPU subexecutors");
    PRINT_EV("NUM_GPU_SE", numGpuSubExecs,
             std::string("Using ") + std::to_string(numGpuSubExecs) + " GPU subexecutors");
669
670
671
672
    PRINT_EV("P2P_MODE", p2pMode,
             std::string("Running ") + (p2pMode == 1 ? "Unidirectional" :
                                        p2pMode == 2 ? "Bidirectional"  :
                                                       "Unidirectional + Bidirectional"));
673
674
675
    PRINT_EV("USE_FINE_GRAIN", useFineGrain,
             std::string("Using ") + (useFineGrain ? "fine" : "coarse") + "-grained memory");

676
677
678
679
680
    PRINT_EV("USE_GPU_DMA", useDmaCopy,
             std::string("Using GPU-") + (useDmaCopy ? "DMA" : "GFX") + " as GPU executor");
    PRINT_EV("USE_REMOTE_READ", useRemoteRead,
             std::string("Using ") + (useRemoteRead ? "DST" : "SRC") + " as executor");
    printf("\n");
gilbertlee-amd's avatar
gilbertlee-amd committed
681
682
  }

Gilbert Lee's avatar
Gilbert Lee committed
683
684
685
  // Display env var settings
  void DisplaySweepEnvVars() const
  {
686
    DisplayEnvVars();
687
    if (hideEnv) return;
688

Gilbert Lee's avatar
Gilbert Lee committed
689
    if (!outputToCsv)
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
      printf("[Sweep Related]\n");
    PRINT_ES("SWEEP_DST", sweepDst.c_str(),
             std::string("Destination Memory Types to sweep"));
    PRINT_ES("SWEEP_EXE", sweepExe.c_str(),
             std::string("Executor Types to sweep"));
    PRINT_EV("SWEEP_MAX", sweepMax,
             std::string("Max simultaneous transfers (0 = no limit)"));
    PRINT_EV("SWEEP_MIN", sweepMin,
             std::string("Min simultaenous transfers"));
    PRINT_EV("SWEEP_RAND_BYTES", sweepRandBytes,
             std::string("Using ") + (sweepRandBytes ? "random" : "constant") + " number of bytes per Transfer");
    PRINT_EV("SWEEP_SEED", sweepSeed,
             std::string("Random seed set to ") + std::to_string(sweepSeed));
    PRINT_ES("SWEEP_SRC", sweepSrc.c_str(),
             std::string("Source Memory Types to sweep"));
    PRINT_EV("SWEEP_TEST_LIMIT", sweepTestLimit,
             std::string("Max number of tests to run during sweep (0 = no limit)"));
    PRINT_EV("SWEEP_TIME_LIMIT", sweepTimeLimit,
             std::string("Max number of seconds to run sweep for  (0 = no limit)"));
    PRINT_EV("SWEEP_XGMI_MAX", sweepXgmiMax,
             std::string("Max number of XGMI hops for Transfers (-1 = no limit)"));
    PRINT_EV("SWEEP_XGMI_MIN", sweepXgmiMin,
             std::string("Min number of XGMI hops for Transfers"));
    printf("\n");
  }
Gilbert Lee's avatar
Gilbert Lee committed
715

716
717
718
719
720
721
722
723
  void DisplayA2AEnvVars() const
  {
    DisplayEnvVars();
    if (hideEnv) return;
    if (!outputToCsv)
      printf("[AllToAll Related]\n");
    PRINT_EV("A2A_DIRECT", a2aDirect,
             std::string(a2aDirect ? "Only using direct links" : "Full all-to-all"));
724
725
    PRINT_EV("USE_FINE_GRAIN", useFineGrain,
             std::string("Using ") + (useFineGrain ? "fine" : "coarse") + "-grained memory");
726
727
728
    PRINT_EV("USE_REMOTE_READ", useRemoteRead,
             std::string("Using ") + (useRemoteRead ? "DST" : "SRC") + " as executor");

729
730
731
    printf("\n");
  }

732
733
734
735
736
737
738
739
740
741
  void DisplaySchmooEnvVars() const
  {
    DisplayEnvVars();
    if (hideEnv) return;
    if (!outputToCsv)
      printf("[Schmoo Related]\n");
    PRINT_EV("USE_FINE_GRAIN", useFineGrain,
             std::string("Using ") + (useFineGrain ? "fine" : "coarse") + "-grained memory");
  }

Gilbert Lee's avatar
Gilbert Lee committed
742
  // Helper function that gets parses environment variable or sets to default value
Gilbert Lee's avatar
Gilbert Lee committed
743
  static int GetEnvVar(std::string const& varname, int defaultValue)
Gilbert Lee's avatar
Gilbert Lee committed
744
745
746
747
748
  {
    if (getenv(varname.c_str()))
      return atoi(getenv(varname.c_str()));
    return defaultValue;
  }
Gilbert Lee's avatar
Gilbert Lee committed
749
750
751
752
753
754
755

  static std::string GetEnvVar(std::string const& varname, std::string const& defaultValue)
  {
    if (getenv(varname.c_str()))
      return getenv(varname.c_str());
    return defaultValue;
  }
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799

  std::string GetCuMaskDesc() const
  {
    std::vector<std::pair<int, int>> runs;

    bool inRun = false;
    std::pair<int, int> curr;
    int used = 0;
    for (int i = 0; i < cuMask.size(); i++)
    {
      for (int j = 0; j < 32; j++)
      {
        if (cuMask[i] & (1 << j))
        {
          used++;
          if (!inRun)
          {
            inRun = true;
            curr.first = i * 32 + j;
          }
        }
        else
        {
          if (inRun)
          {
            inRun = false;
            curr.second = i * 32 + j - 1;
            runs.push_back(curr);
          }
        }
      }
    }
    if (inRun)
      curr.second = cuMask.size() * 32 - 1;

    std::string result = "CUs used: (" + std::to_string(used) + ") ";
    for (int i = 0; i < runs.size(); i++)
    {
      if (i) result += ",";
      if (runs[i].first == runs[i].second) result += std::to_string(runs[i].first);
      else result += std::to_string(runs[i].first) + "-" + std::to_string(runs[i].second);
    }
    return result;
  }
Gilbert Lee's avatar
Gilbert Lee committed
800
801
802
};

#endif