name: PR Test (AMD)

on:
  push:
    branches: [ main ]
    paths:
      - "python/**"
      - "scripts/ci/**"
      - "test/**"
      - "sgl-kernel/**"
      - ".github/workflows/pr-test-amd.yml"
  pull_request:
    branches: [ main ]
    paths:
      - "python/**"
      - "scripts/ci/**"
      - "test/**"
      - "sgl-kernel/**"
      - ".github/workflows/pr-test-amd.yml"
  workflow_dispatch:

concurrency:
  group: pr-test-amd-${{ github.ref }}
  cancel-in-progress: true

jobs:
  accuracy-test-1-gpu-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
      github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-1, linux-mi325-gpu-1]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Evaluate Accuracy
        timeout-minutes: 30
        run: |
          bash scripts/ci/amd_ci_exec.sh -e SGLANG_USE_AITER=0 python3 test_eval_accuracy_large.py
          bash scripts/ci/amd_ci_exec.sh python3 test_eval_fp8_accuracy.py
          bash scripts/ci/amd_ci_exec.sh python3 models/test_qwen_models.py

  accuracy-test-2-gpu-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
        github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-2, linux-mi325-gpu-2]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Evaluate accuracy (TP=2)
        timeout-minutes: 30
        run: |
          bash scripts/ci/amd_ci_exec.sh -e SGLANG_USE_AITER=0 python3 test_moe_eval_accuracy_large.py

  mla-test-1-gpu-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
      github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-1, linux-mi325-gpu-1]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: MLA TEST
        timeout-minutes: 30
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 test_mla.py

  performance-test-1-gpu-part-1-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
        github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-1, linux-mi325-gpu-1]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Benchmark single latency
        timeout-minutes: 20
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_small
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_default

      - name: Benchmark online latency
        timeout-minutes: 15
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_online_latency_default

      - name: Benchmark offline throughput
        timeout-minutes: 15
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default

      - name: Benchmark offline throughput (Non-streaming, small batch size)
        timeout-minutes: 15
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_non_stream_small_batch_size

  performance-test-1-gpu-part-2-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
        github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-1, linux-mi325-gpu-1]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Benchmark offline throughput (w/o RadixAttention)
        timeout-minutes: 15
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_without_radix_cache

      - name: Benchmark offline throughput (w/ Triton)
        timeout-minutes: 15
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_with_triton_attention_backend

      - name: Benchmark offline throughput (w/ FP8)
        timeout-minutes: 15
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default_fp8

  bench-test-2-gpu-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
      github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-2, linux-mi325-gpu-2]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Benchmark dummy grok (TP=2)
        timeout-minutes: 30
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 models/test_dummy_grok_models.py

      - name: Benchmark single latency (TP=2)
        timeout-minutes: 25
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_moe_tp2_bs1

      - name: Benchmark single latency + torch.compile (TP=2)
        timeout-minutes: 25
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_torch_compile_tp2_bs1

      - name: Benchmark offline throughput (TP=2)
        timeout-minutes: 25
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_default

      - name: Benchmark offline throughput (w/o RadixAttention) (TP=2)
        timeout-minutes: 25
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_without_radix_cache

  unit-test-backend-1-gpu-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
      github.event.pull_request.draft == false
    strategy:
      fail-fast: false
      matrix:
        runner: [linux-mi300-gpu-1, linux-mi325-gpu-1]
        part: [0, 1, 2, 3, 4, 5, 6, 7]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Run test
        timeout-minutes: 50
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 8

  unit-test-backend-2-gpu-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
      github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-2, linux-mi325-gpu-2]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Run test
        timeout-minutes: 40
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-2-gpu-amd

  unit-test-backend-8-gpu-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
      github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-8]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Run test
        timeout-minutes: 60
        run: |
          bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-8-gpu-amd --timeout-per-file 3600

  unit-test-backend-8-gpu-CAR-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
      github.event.pull_request.draft == false
    strategy:
      matrix:
        runner: [linux-mi300-gpu-8]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: bash scripts/ci/amd_ci_install_dependency.sh

      - name: Run CustomAllReduce test
        timeout-minutes: 20
        run: |
          bash scripts/ci/amd_ci_exec.sh -e CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python3 -m unittest test_custom_allreduce.TestCustomAllReduce

  unit-test-sgl-kernel-amd:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') &&
      github.event.pull_request.draft == false
    strategy:
      fail-fast: false
      matrix:
        runner: [linux-mi300-gpu-1, linux-mi325-gpu-1]
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Start CI container
        run: bash scripts/ci/amd_ci_start_container.sh
        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Install dependencies
        run: |
          bash scripts/ci/amd_ci_install_dependency.sh

      - name: Run test
        timeout-minutes: 14
        run: |
          docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_moe_align.py
          docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_moe_topk_softmax.py
          docker exec -w /sglang-checkout/sgl-kernel/tests/speculative ci_sglang python3 -m pytest test_eagle_utils.py
          docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_apply_token_bitmask_inplace.py
          docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_activation.py
          docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_kvcacheio.py

  pr-test-amd-finish:
    if: always()
    needs: [
      accuracy-test-1-gpu-amd, mla-test-1-gpu-amd, bench-test-2-gpu-amd,
      accuracy-test-2-gpu-amd, performance-test-1-gpu-part-1-amd, performance-test-1-gpu-part-2-amd,
      unit-test-backend-1-gpu-amd, unit-test-backend-2-gpu-amd, unit-test-backend-8-gpu-amd,
      unit-test-sgl-kernel-amd
    ]
    runs-on: ubuntu-latest
    steps:
      - name: Check all dependent job statuses
        run: |
          results=(${{ join(needs.*.result, ' ') }})
          for result in "${results[@]}"; do
            if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then
              echo "Job failed with result: $result"
              exit 1
            fi
          done
          echo "All jobs completed successfully"
          exit 0