Release - SuperBench v0.9.0 (#558)

**Description**
Cherry-pick bug fixes from v0.9.0 to main.

**Major Revision**
- CI/CD: pipeline - clean more disk space to fix rocm building image
pipeline(#555 )
- Benchmarks: bug fix - use absolute path for input file in
DirectXEncodingLatency(#554)
- CI/CD - add push win docker image on release branch in pipeline (#552)
- Docs - Upgrade version and release note(#557)

.github/workflows/build-image.yml

@@ -64,6 +64,17 @@ jobs:
           sudo apt-get clean
           sudo docker rmi $(sudo docker images --format "{{.Repository}}:{{.Tag}}" --filter=reference="node" --filter=reference="buildpack-deps")
           df -h
+      - name: Free Up GitHub Actions Ubuntu Runner Disk Space 🔧
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # This might remove tools that are actually needed, if set to "true" but frees about 6 GB
+          tool-cache: false
+          # All of these default to true, but feel free to set to "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
       - name: Prepare metadata
         id: metadata
         run: |

.github/workflows/build-win.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   docker:
-    name: Docker build win2004
+    name: Docker build win directx12
     runs-on: [self-hosted, windows, x64, win2004]
     steps:
     - name: Checkout
@@ -24,6 +24,25 @@ jobs:
         docker system prune -a -f
         docker volume prune -a -f
       shell: pwsh
+    - name: Set TAG variable based on the branch
+      run: |
+        if ($env:GITHUB_EVENT_NAME -match "release") {
+          $version = $env:GITHUB_REF.Substring($env:GITHUB_REF.LastIndexOf('/') + 1)
+          echo "TAG=superbench/superbench:$version-directx12" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        } elseif ($env:GITHUB_REF -match "refs/heads/release/(.*)") {
+          $version = $Matches[1]
+          echo "TAG=superbench/release:$version-directx12" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        } elseif ($env:GITHUB_BASEREF -match "release/(.*)"){
+          $version = $Matches[1]
+          echo "TAG=superbench/release:$version-directx12" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        } else {
+          echo "TAG=superbench/main:directx12" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        }
+      shell: pwsh
+      env:
+        GITHUB_REF: ${{ github.ref }}
+        GITHUB_BASEREF: ${{ github.base_ref }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
     - name: Build Docker image
       working-directory: .
       shell: pwsh
@@ -37,7 +56,7 @@ jobs:
           --isolation=process `
           --tag $env:TAG .
       env:
-        TAG: superbench/main:win2004
+        TAG: ${{ env.TAG }}
     - name: Push Docker image
       if: ${{ github.event_name != 'pull_request' }}
       shell: pwsh
@@ -46,7 +65,7 @@ jobs:
         docker push $env:TAG
         docker logout
       env:
-        TAG: superbench/main:win2004
+        TAG: ${{ env.TAG }}
         USER: ${{ secrets.DOCKERHUB_USERNAME }}
         PASS: ${{ secrets.DOCKERHUB_TOKEN }}
     - name: Add bash to PATH
@@ -64,7 +83,9 @@ jobs:
         docker run --rm `
         --isolation process `
         --device class/5B45201D-F2F2-4F3B-85BB-30FF1F953599 `
-        -e CI=true $ci_env -e SB_TEST_CUDA="0" -e SB_TEST_ROCM="0" -e SB_TEST_PYTORCH="0" -e SB_TEST_DIRECTX="1" -e CODECOV_TOKEN superbench/main:win2004 cmd /c $command
+        -v C:/Windows/System32/DriverStore:C:/Windows/System32/DriverStore `
+        -e CI=true $ci_env -e SB_TEST_CUDA="0" -e SB_TEST_ROCM="0" -e SB_TEST_PYTORCH="0" -e SB_TEST_DIRECTX="1" -e CODECOV_TOKEN --entrypoint "cmd" $env:TAG "/c python dockerfile/directx/enable-graphics-apis.py && cmd /c $command"
       shell: pwsh
       env:
+        TAG: ${{ env.TAG }}
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

README.md

@@ -15,7 +15,7 @@
 
 __SuperBench__ is a validation and profiling tool for AI infrastructure.
 
-📢 [v0.8.0](https://github.com/microsoft/superbenchmark/releases/tag/v0.8.0) has been released!
+📢 [v0.9.0](https://github.com/microsoft/superbenchmark/releases/tag/v0.9.0) has been released!
 
 ## _Check [aka.ms/superbench](https://aka.ms/superbench) for more details._
 

docs/getting-started/installation.mdx

@@ -61,7 +61,7 @@ You can clone the source from GitHub and build it.
 :::note Note
 You should checkout corresponding tag to use release version, for example,
 
-`git clone -b v0.8.0 https://github.com/microsoft/superbenchmark`
+`git clone -b v0.9.0 https://github.com/microsoft/superbenchmark`
 :::
 
 ```bash

docs/getting-started/run-superbench.md

@@ -27,7 +27,7 @@ sb deploy -f remote.ini --host-password [password]
 :::note Note
 You should deploy corresponding Docker image to use release version, for example,
 
-`sb deploy -f local.ini -i superbench/superbench:v0.8.0-cuda12.1`
+`sb deploy -f local.ini -i superbench/superbench:v0.9.0-cuda12.1`
 
 You should note that version of git repo only determines version of sb CLI, and not the sb container. You should define the container version even if you specified a release version for the git clone.
 

docs/superbench-config.mdx

@@ -70,7 +70,7 @@ superbench:
 <TabItem value='example'>
 
 ```yaml
-version: v0.8
+version: v0.9
 superbench:
   enable: benchmark_1
   monitor:

docs/user-tutorial/container-images.mdx

@@ -23,12 +23,15 @@ available tags are listed below for all stable versions.
   values={[
     {label: 'CUDA', value: 'cuda'},
     {label: 'ROCm', value: 'rocm'},
+    {label: 'DirectX', value: 'directx'},
   ]
 }>
 <TabItem value='cuda'>
 
 | Tag               | Description                        |
 |-------------------|------------------------------------|
+| v0.9.0-cuda12.1   | SuperBench v0.9.0 with CUDA 12.1   |
+| v0.9.0-cuda11.1.1 | SuperBench v0.9.0 with CUDA 11.1.1 |
 | v0.8.0-cuda12.1   | SuperBench v0.8.0 with CUDA 12.1   |
 | v0.8.0-cuda11.1.1 | SuperBench v0.8.0 with CUDA 11.1.1 |
 | v0.7.0-cuda11.8   | SuperBench v0.7.0 with CUDA 11.8   |
@@ -45,6 +48,10 @@ available tags are listed below for all stable versions.
 
 | Tag                           | Description                                      |
 |-------------------------------|--------------------------------------------------|
+| v0.9.0-rocm5.1.3              | SuperBench v0.9.0 with ROCm 5.1.3                |
+| v0.9.0-rocm5.1.1              | SuperBench v0.9.0 with ROCm 5.1.1                |
+| v0.9.0-rocm5.0.1              | SuperBench v0.9.0 with ROCm 5.0.1                |
+| v0.9.0-rocm5.0                | SuperBench v0.9.0 with ROCm 5.0                  |
 | v0.8.0-rocm5.1.3              | SuperBench v0.8.0 with ROCm 5.1.3                |
 | v0.8.0-rocm5.1.1              | SuperBench v0.8.0 with ROCm 5.1.1                |
 | v0.8.0-rocm5.0.1              | SuperBench v0.8.0 with ROCm 5.0.1                |
@@ -66,5 +73,12 @@ available tags are listed below for all stable versions.
 | v0.3.0-rocm4.2-pytorch1.7.0   | SuperBench v0.3.0 with ROCm 4.2, PyTorch 1.7.0   |
 | v0.3.0-rocm4.0-pytorch1.7.0   | SuperBench v0.3.0 with ROCm 4.0, PyTorch 1.7.0   |
 
+</TabItem>
+<TabItem value='directx'>
+
+| Tag                           | Description                                      |
+|-------------------------------|--------------------------------------------------|
+| v0.9.0-directx12              | SuperBench v0.9.0 with DirectX12, Windows10-2004 |
+
 </TabItem>
 </Tabs>

docs/user-tutorial/data-diagnosis.md

@@ -65,7 +65,7 @@ superbench:
 example:
 ```yaml
 # SuperBench rules
-version: v0.8
+version: v0.9
 superbench:
   rules:
     failure-rule:

docs/user-tutorial/result-summary.md

@@ -58,7 +58,7 @@ superbench:
 
 ```yaml title="Example"
 # SuperBench rules
-version: v0.8
+version: v0.9
 superbench:
   rules:
     kernel_launch:

superbench/__init__.py

@@ -6,5 +6,5 @@
 Provide hardware and software benchmarks for AI systems.
 """
 
-__version__ = '0.8.0'
+__version__ = '0.9.0'
 __author__ = 'Microsoft'

superbench/benchmarks/micro_benchmarks/directx_gpu_encoding_latency.py

@@ -98,11 +98,11 @@ def _preprocess(self):
         command += f' -HEIGHT {self._args.height}'
         command += f' -WIDTH {self._args.width}'
         if self._args.input_file is not None:
-            command += f' -INPUT {self._args.input_file}'
+            command += f' -INPUT {os.path.abspath(self._args.input_file)}'
         else:
             if not os.path.exists(f'{self._test_file}'):
                 create_nv12_file(self._test_file, self._args.frames, self._args.width, self._args.height)
-            command += f' -INPUT {self._test_file}'
+            command += f' -INPUT {os.path.abspath(self._test_file)}'
         if self._args.output_file is not None:
             command += f' -OUTPUT {self._args.output_file}'
         command += f' -OUTPUT_HEIGHT {self._args.output_height}'

superbench/config/amd_mi100_hpe.yaml

@@ -3,7 +3,7 @@
 # Server:
 #   - Product: HPE Apollo 6500
 
-version: v0.8
+version: v0.9
 superbench:
   enable: null
   var:

superbench/config/amd_mi100_z53.yaml

@@ -4,7 +4,7 @@
 #   - Product: G482-Z53
 #   - Link: https://www.gigabyte.cn/FileUpload/Global/MicroSite/553/G482-Z53.html
 
-version: v0.8
+version: v0.9
 superbench:
   enable: null
   var:

superbench/config/azure/inference/standard_nc64as_t4_v3.yaml

-version: v0.8
+version: v0.9
 superbench:
   enable: null
   monitor:

superbench/config/azure/inference/standard_nc96ads_a100_v4.yaml

-version: v0.8
+version: v0.9
 superbench:
   enable: null
   monitor:

superbench/config/azure/inference/standard_nv18ads_a10_v5.yaml

-version: v0.8
+version: v0.9
 superbench:
   enable: null
   monitor:

superbench/config/azure_ndmv4.yaml

@@ -3,7 +3,7 @@
 # Azure NDm A100 v4
 # reference: https://docs.microsoft.com/en-us/azure/virtual-machines/ndm-a100-v4-series
 
-version: v0.8
+version: v0.9
 superbench:
   enable: null
   monitor:

superbench/config/azure_ndv4.yaml

 # SuperBench Config
-version: v0.8
+version: v0.9
 superbench:
   enable: null
   monitor:

superbench/config/default.yaml

 # SuperBench Config
-version: v0.8
+version: v0.9
 superbench:
   enable: null
   monitor:

third_party/Makefile

@@ -157,5 +157,5 @@ directx_amf_encoding_latency:
 		curl -L -o vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe && echo "Downloaded vs_buildtools.exe" && \
 		start /wait vs_buildtools.exe --quiet --wait --norestart --nocache --installPath C:/temp/BuildTools --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.VC.ATLMFC --includeRecommended  && echo "Installed VS Build Tools" && \
 		del vs_buildtools.exe && echo "Deleted vs_buildtools.exe" && \
-		"C:\temp\BuildTools\MSBuild\Current\Bin\MSBuild.exe" "AMF\amf\public\samples\CPPSamples_vs2019.sln" /t:EncoderLatency /p:Configuration=Release /p:OutDir="%SB_MICRO_PATH%\bin" \
+		"C:\temp\BuildTools\MSBuild\Current\Bin\MSBuild.exe" "AMF\amf\public\samples\CPPSamples_vs2019.sln" /t:EncoderLatency /p:Platform=x64 /p:Configuration=Release /p:OutDir="%SB_MICRO_PATH%\bin" \
 	)