Enable latency test in ib traffic validation distributed benchmark (#396)

Enable ib latency test in ib traffic validation distributed benchmark

As Perftest supports CUDA only in BW tests (Refer [perftest source code](https://github.com/linux-rdma/perftest/blob/23f7f8a56892bd9e00b6a4e8bd4bbfdbe122af47/src/perftest_parameters.c#L1652))
Remove `--use_cuda` option from cmd prefix if the test command is bw related

superbench/benchmarks/micro_benchmarks/ib_validation_performance.py

@@ -297,15 +297,18 @@ def __prepare_general_ib_command_params(self):
         # Add GPUDirect for ib command
         gpu_dev = ''
         if self._args.gpu_dev is not None:
-            gpu = GPU()
-            if gpu.vendor == 'nvidia':
-                gpu_dev = f'--use_cuda={self._args.gpu_dev}'
-            elif gpu.vendor == 'amd':
-                gpu_dev = f'--use_rocm={self._args.gpu_dev}'
-            else:
-                self._result.set_return_code(ReturnCode.INVALID_ARGUMENT)
-                logger.error('No GPU found - benchmark: {}'.format(self._name))
-                return False
+            if 'bw' in self._args.command:
+                gpu = GPU()
+                if gpu.vendor == 'nvidia':
+                    gpu_dev = f'--use_cuda={self._args.gpu_dev}'
+                elif gpu.vendor == 'amd':
+                    gpu_dev = f'--use_rocm={self._args.gpu_dev}'
+                else:
+                    self._result.set_return_code(ReturnCode.INVALID_ARGUMENT)
+                    logger.error('No GPU found - benchmark: {}'.format(self._name))
+                    return False
+            elif 'lat' in self._args.command:
+                logger.warning('Wrong configuration: Perftest supports CUDA/ROCM only in BW tests')
         # Generate ib command params
         command_params = f'-F -n {self._args.iters} -d {self._args.ib_dev} {msg_size} {gpu_dev}'
         command_params = f'{command_params.strip()} --report_gbits'

superbench/benchmarks/micro_benchmarks/ib_validation_performance/ib_validation_performance.cc

@@ -260,13 +260,28 @@ void gather_hostnames(vector<string> &hostnames, string filename) {
 }
 
 // Parse raw output of ib command
-// TODO: does not work latency tests
+// Sample of ib bw command raw
+// #bytes     #iterations BW peak[Gb/sec]    BW average[Gb/sec]  MsgRate[Mpps]
+// 8388608    5000            196.08             195.76            0.002917
+// Sample of ib latency command raw output
+// #bytes  #iterations    t_min    t_max  t_typical   t_avg    t_stdev  99% percentile   99.9% percentile
+// 8388608    5000        581.27   876.26   594.87    595.50     3.33       601.65          621.14
+// parsed result:
+// 195.76 (BW average)
+// 595.50 (t_avg)
 float process_raw_output(string output) {
     float res = -1.0;
     try {
+        string pattern;
         vector<string> lines;
         boost::split(lines, output, boost::is_any_of("\n"), boost::token_compress_on);
-        regex re("\\d+\\s+\\d+\\s+\\d+\\.\\d+\\s+(\\d+\\.\\d+)\\s+\\d+\\.\\d+");
+        if (output.find("BW") != string::npos) {
+            pattern = "\\d+\\s+\\d+\\s+\\d+\\.\\d+\\s+(\\d+\\.\\d+)\\s+\\d+\\.\\d+";
+        } else {
+            pattern = "\\d+\\s+\\d+\\s+\\d+\\.\\d+\\s+\\d+\\.\\d+\\s+\\d+\\.\\d+"
+                      "\\s+(\\d+\\.\\d+)\\s+\\d+\\.\\d+\\s+\\d+\\.\\d+\\s+\\d+\\.\\d+";
+        }
+        regex re(pattern);
         for (string line : lines) {
             smatch m;
             if (regex_search(line, m, re))

tests/benchmarks/micro_benchmarks/test_ib_traffic_performance.py

@@ -206,6 +206,17 @@ def test_ib_traffic_performance(self, mock_gpu):
         command = benchmark._bin_name + benchmark._commands[0].split(benchmark._bin_name)[1]
         assert (command == expect_command)
 
+        parameters = '--command ib_read_lat --ib_dev mlx5_0 --iters 2000 --msg_size 33554432 ' + \
+            '--pattern one-to-one --hostfile hostfile --gpu_dev 0'
+        mock_gpu.return_value = 'nvidia'
+        benchmark = benchmark_class(benchmark_name, parameters=parameters)
+        ret = benchmark._preprocess()
+        expect_command = "ib_validation --cmd_prefix '" + benchmark._args.bin_dir + \
+            "/ib_read_lat -F -n 2000 -d mlx5_0 -s 33554432 --report_gbits' " + \
+            f'--timeout 120 --hostfile hostfile --input_config {os.getcwd()}/config.txt'
+        command = benchmark._bin_name + benchmark._commands[0].split(benchmark._bin_name)[1]
+        assert (command == expect_command)
+
         # Custom config
         config = ['0,1', '1,0;0,1', '0,1;1,0', '1,0;0,1']
         with open('test_config.txt', 'w') as f: