name: Accuracy Test

on:
  push:
    branches: [ main ]
    paths:
      - "python/sglang/**"
      - "test/**"
      - ".github/workflows/accuracy-test.yml"
  pull_request:
    branches: [ main ]
    paths:
      - "python/sglang/**"
      - "test/**"
      - ".github/workflows/accuracy-test.yml"
  workflow_dispatch:

concurrency:
  group: accuracy-test-${{ github.ref }}
  cancel-in-progress: true

jobs:
  accuracy-test:
    if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request'
    runs-on: accuracy

    steps:
    - name: Checkout code
      uses: actions/checkout@v3

    - name: Install dependencies
      run: |
        source $HOME/venv/bin/activate
        echo "$HOME/venv/bin" >> $GITHUB_PATH

        pip install --upgrade pip
        pip install -e "python[all]"
        pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.4/ --force-reinstall

        git clone https://github.com/merrymercy/human-eval.git
        cd human-eval
        pip install -e .

    - name: Evaluate Accuracy
      run: |
        cd test/srt
        python3 test_eval_accuracy_large.py
      timeout-minutes: 10
