Benchmarks: micro benchmarks - add python code for DirectXGPUEncodingLatency (#548)

**Description**
add python code for DirectXGPUEncodingLatency.

dockerfile/directx12.dockerfile

@@ -64,5 +64,5 @@ RUN make -C third_party directx_amd
 
 # Run the entrypoint script for enabling vendor-specific graphics APIs
 RUN powershell -Command "Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope LocalMachine -Force"
-CMD [ "python", "dockerfile/directx/enable-graphics-apis.py" ]
-ENTRYPOINT [ "cmd.exe" ]
+ENTRYPOINT [ "python", "dockerfile/directx/enable-graphics-apis.py" ]
+CMD [ "cmd.exe" ]

superbench/benchmarks/micro_benchmarks/__init__.py

@@ -31,6 +31,7 @@
 from superbench.benchmarks.micro_benchmarks.sharding_matmul import ShardingMatmul
 from superbench.benchmarks.micro_benchmarks.tcp_connectivity import TCPConnectivityBenchmark
 from superbench.benchmarks.micro_benchmarks.tensorrt_inference_performance import TensorRTInferenceBenchmark
+from superbench.benchmarks.micro_benchmarks.directx_gpu_encoding_latency import DirectXGPUEncodingLatency
 from superbench.benchmarks.micro_benchmarks.directx_gpu_copy_performance import DirectXGPUCopyBw
 from superbench.benchmarks.micro_benchmarks.directx_mem_bw_performance import DirectXGPUMemBw
 from superbench.benchmarks.micro_benchmarks.directx_gemm_flops_performance import DirectXGPUCoreFlops
@@ -64,6 +65,7 @@
     'ShardingMatmul',
     'TCPConnectivityBenchmark',
     'TensorRTInferenceBenchmark',
+    'DirectXGPUEncodingLatency',
     'DirectXGPUCopyBw',
     'DirectXGPUMemBw',
     'DirectXGPUCoreFlops',

superbench/benchmarks/micro_benchmarks/directx_gpu_encoding_latency.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Module of the DirectXGPUEncodingLatency benchmarks."""
+
+import os
+
+from superbench.common.utils import logger
+from superbench.benchmarks import BenchmarkRegistry, Platform
+from superbench.benchmarks.micro_benchmarks import MicroBenchmarkWithInvoke
+
+
+def create_nv12_file(file_name, num_frames, width, height):
+    """Create a NV12 file with the specified name, number of frames, width, and height."""
+    import numpy as np
+    # Generate a Y plane of width x height with values from 0-255
+    y_plane = np.random.randint(0, 256, (height, width), dtype=np.uint8)
+    # Generate a UV plane of width x height/2 with values from 0-255
+    uv_plane = np.random.randint(0, 256, (height // 2, width), dtype=np.uint8)
+    # Create the file
+    with open(f'{file_name}', 'wb') as f:
+        for _ in range(num_frames):
+            # Write the Y plane and UV plane to the file
+            f.write(y_plane.tobytes())
+            f.write(uv_plane.tobytes())
+
+
+class DirectXGPUEncodingLatency(MicroBenchmarkWithInvoke):
+    """The DirectXGPUEncodingLatency benchmark class."""
+    def __init__(self, name, parameters=''):
+        """Constructor."""
+        super().__init__(name, parameters)
+        self._bin_name = 'EncoderLatency.exe'
+        self._test_file = 'test_directx_gpu_encoding_latency.nv12'
+
+    def add_parser_arguments(self):
+        """Add the specified arguments."""
+        super().add_parser_arguments()
+        self._parser.add_argument(
+            '--algo',
+            type=str,
+            choices=['ASAP', 'OneInOne'],
+            default='ASAP',
+            required=False,
+            help='The algorithm to use for encoding'
+        )
+        self._parser.add_argument(
+            '--codec',
+            type=str,
+            choices=['AVC', 'H264', 'HEVC', 'H265', 'AV1'],
+            default='H265',
+            required=False,
+            help='The codec to use for encoding'
+        )
+        self._parser.add_argument(
+            '--format',
+            type=str,
+            choices=['RGBA_F16', 'R10G10B10A2', 'NV12', 'P010'],
+            default='NV12',
+            required=False,
+            help='The format to use for encoding'
+        )
+        self._parser.add_argument(
+            '--frames', type=int, default=500, required=False, help='The number of frames to encode'
+        )
+        self._parser.add_argument(
+            '--height', type=int, default=720, required=False, help='The height of the input video'
+        )
+        self._parser.add_argument(
+            '--width', type=int, default=1080, required=False, help='The width of the input video'
+        )
+        self._parser.add_argument('--input_file', type=str, default=None, required=False, help='The input video file')
+        self._parser.add_argument('--output_file', type=str, default=None, required=False, help='The output video file')
+        self._parser.add_argument(
+            '--output_height', type=int, default=720, required=False, help='The height of the output video'
+        )
+        self._parser.add_argument(
+            '--output_width', type=int, default=1080, required=False, help='The width of the output video'
+        )
+        self._parser.add_argument(
+            '--vcn', type=int, choices=[0, 1], default=0, required=False, help='The VCN instance to use for encoding'
+        )
+
+    def _preprocess(self):
+        """Preprocess/preparation operations before the benchmarking.
+
+        Return:
+            True if _preprocess() succeed.
+        """
+        if not super()._preprocess():
+            return False
+
+        command = os.path.join(self._args.bin_dir, self._bin_name)
+        command += f' -ALGORITHM {self._args.algo}'
+        command += f' -CODEC {self._args.codec}'
+        command += f' -FORMAT {self._args.format}'
+        command += f' -FRAMES {self._args.frames}'
+        command += f' -HEIGHT {self._args.height}'
+        command += f' -WIDTH {self._args.width}'
+        if self._args.input_file is not None:
+            command += f' -INPUT {self._args.input_file}'
+        else:
+            if not os.path.exists(f'{self._test_file}'):
+                create_nv12_file(self._test_file, self._args.frames, self._args.width, self._args.height)
+            command += f' -INPUT {self._test_file}'
+        if self._args.output_file is not None:
+            command += f' -OUTPUT {self._args.output_file}'
+        command += f' -OUTPUT_HEIGHT {self._args.output_height}'
+        command += f' -OUTPUT_WIDTH {self._args.output_width}'
+        command += f' -VCNINSTANCE {self._args.vcn}'
+        self._commands.append(command)
+
+        return True
+
+    def _process_raw_result(self, cmd_idx, raw_output):
+        """Function to parse raw results and save the summarized results.
+
+          self._result.add_raw_data() and self._result.add_result() need to be called to save the results.
+
+        Args:
+            cmd_idx (int): the index of command corresponding with the raw_output.
+            raw_output (str): raw output string of the micro-benchmark.
+
+        Return:
+            True if the raw output string is valid and result can be extracted.
+        """
+        self._result.add_raw_data('raw_output', raw_output, self._args.log_raw_data)
+
+        content = raw_output.splitlines()
+        metrics = {}
+
+        try:
+            for line in content:
+                if 'Total' in line:
+                    metrics['fps'] = float(line.split('=')[3].strip().strip('frames').split()[0])
+                if 'Latency' in line and 'min' in line.lower():
+                    metrics['min_lat'] = float(line.split('=')[1].split(',')[1].strip('ms').strip())
+                    metrics['max_lat'] = float(line.split('=')[1].split(',')[2].strip('ms').strip())
+                if 'Latency' in line and 'average' in line.lower():
+                    metrics['avg_lat'] = float(line.split('=')[1].strip('ms').strip())
+        except Exception as e:
+            logger.error(
+                'The result format is invalid - benchmark: {}, raw output: {}, error: {}'.format(
+                    self._name, raw_output, str(e)
+                )
+            )
+            return False
+
+        for metric, value in metrics.items():
+            self._result.add_result(metric, value)
+
+        return True
+
+
+BenchmarkRegistry.register_benchmark(
+    'directx-gpu-encoding-latency', DirectXGPUEncodingLatency, platform=Platform.DIRECTX
+)

tests/benchmarks/micro_benchmarks/test_directx_gpu_encoding_latency.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""Tests for DirectXGPUEncodingLatency benchmark."""
+
+import numbers
+
+from tests.helper import decorator
+from superbench.benchmarks import BenchmarkRegistry, BenchmarkType, ReturnCode, Platform
+
+
+@decorator.directx_test
+def test_directx_gpuencodinglatency():
+    """Test DirectXGPUEncodingLatency benchmark."""
+    context = BenchmarkRegistry.create_benchmark_context(
+        'directx-gpu-encoding-latency',
+        platform=Platform.DIRECTX,
+        parameters=r'--algo ASAP --codec H265 --format NV12 --frames 500' +
+        r' --height 720 --width 1080 --output_height 720 --output_width 1080 --vcn 0'
+    )
+
+    assert (BenchmarkRegistry.is_benchmark_context_valid(context))
+
+    benchmark = BenchmarkRegistry.launch_benchmark(context)
+
+    # Check basic information.
+    assert (benchmark)
+    assert (benchmark.name == 'directx-gpu-encoding-latency')
+    assert (benchmark.type == BenchmarkType.MICRO)
+
+    # Check parameters specified in BenchmarkContext.
+    assert (benchmark._args.algo == 'ASAP')
+    assert (benchmark._args.codec == 'H265')
+    assert (benchmark._args.format == 'NV12')
+    assert (benchmark._args.frames == 500)
+    assert (benchmark._args.height == 720)
+    assert (benchmark._args.width == 1080)
+    assert (benchmark._args.output_height == 720)
+    assert (benchmark._args.output_width == 1080)
+    assert (benchmark._args.vcn == 0)
+
+    # Check results and metrics.
+    assert (benchmark._args.run_count == 1)
+    assert (benchmark.return_code == ReturnCode.SUCCESS)
+    assert ('raw_output' in benchmark.raw_data)
+    assert (len(benchmark.raw_data['raw_output']) == 1)
+    assert (isinstance(benchmark.raw_data['raw_output'][0], str))
+
+    assert ('fps' in benchmark.result)
+    assert ('min_lat' in benchmark.result)
+    assert ('max_lat' in benchmark.result)
+    assert ('avg_lat' in benchmark.result)
+    assert (isinstance(benchmark.result['fps'][0], numbers.Number))
+    assert (isinstance(benchmark.result['min_lat'][0], numbers.Number))
+    assert (isinstance(benchmark.result['max_lat'][0], numbers.Number))
+    assert (isinstance(benchmark.result['avg_lat'][0], numbers.Number))