name: 'Pytest'
description: 'Run pytest on pre-built container images'
inputs:
  pytest_marks:
    description: 'Pytest marks'
    required: true
    default: 'e2e and vllm and gpu_1 and not slow'
  image_tag:
    description: 'Image Tag to run tests on'
    required: true
  framework:
    description: 'Framework name for test metrics'
    required: false
    default: 'unknown'
  test_type:
    description: 'Test type (unit, e2e, integration)'
    required: false
    default: 'e2e'
  platform_arch:
    description: 'Platform architecture (amd64, arm64)'
    required: false
    default: 'amd64'
  dry_run:
    description: 'Run pytest in dry-run mode (collect tests only, do not execute)'
    required: false
    default: 'false'
  start_minio:
    description: 'Start MinIO service for LoRA tests (true/false)'
    required: false
    default: 'true'
  hf_token:
    required: false
  parallel_mode:
    description: 'Parallelization mode: auto (use all cores), none/0 (sequential), or a number of workers'
    required: false
    default: 'auto'
  dind_as_sidecar:
    description: 'dind runs as a sidecar container (true/false)'
    required: false
    default: 'false'
  cpu_limit:
    description: 'Maximum number of cores available to docker'
    required: false
    default: '10'

runs:
  using: "composite"
  steps:
    - name: Setup Test Environment
      shell: bash
      run: |
        # Setup test directories
        mkdir -p test-results

        # Set platform architecture from input
        PLATFORM_ARCH="${{ inputs.platform_arch }}"
        if [[ -z "${PLATFORM_ARCH}" ]]; then
          PLATFORM_ARCH="amd64"
        fi
        echo "PLATFORM_ARCH=${PLATFORM_ARCH}" >> $GITHUB_ENV
        echo "🏗️  Platform architecture: ${PLATFORM_ARCH}"

    - name: Start MinIO Service
      if: inputs.start_minio == 'true'
      shell: bash
      env:
        MINIO_CONTAINER_NAME: dynamo-minio-test
        MINIO_ACCESS_KEY: minioadmin
        MINIO_SECRET_KEY: minioadmin
      run: |
        # Start MinIO for S3-compatible object storage (used by LoRA tests)
        echo "🗄️  Starting MinIO service..."

        # Remove any existing container
        docker rm -f "${MINIO_CONTAINER_NAME}" 2>/dev/null || true

        docker run -d \
          --name "${MINIO_CONTAINER_NAME}" \
          -p 9000:9000 \
          -p 9001:9001 \
          -e "MINIO_ROOT_USER=${MINIO_ACCESS_KEY}" \
          -e "MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY}" \
          quay.io/minio/minio server /data --console-address ':9001'

        # Wait for MinIO to be ready
        echo "⏳ Waiting for MinIO to be ready..."
        MAX_ATTEMPTS=30
        ATTEMPT=0
        while [ $ATTEMPT -lt $MAX_ATTEMPTS ]; do
          if curl -sf http://localhost:9000/minio/health/live > /dev/null 2>&1; then
            echo "✅ MinIO is ready (attempt $((ATTEMPT + 1)))"
            break
          fi
          ATTEMPT=$((ATTEMPT + 1))
          if [ $ATTEMPT -eq $MAX_ATTEMPTS ]; then
            echo "❌ MinIO failed to start within ${MAX_ATTEMPTS}s"
            echo "📋 Container logs:"
            docker logs "${MINIO_CONTAINER_NAME}" 2>&1 || true
            exit 1
          fi
          sleep 1
        done

    - name: Run tests for runner v1
      if: inputs.dind_as_sidecar == 'false'
      shell: bash
      env:
        NUM_CPUS: ${{ inputs.cpu_limit }}
        CONTAINER_ID: test_${{ github.run_id }}_${{ github.run_attempt }}_${{ github.job }}
        PYTEST_XML_FILE: pytest_test_report.xml
        HF_HOME: /runner/_work/_temp
        HF_TOKEN: ${{ inputs.hf_token }}
      run: |
        # Run pytest with detailed output and JUnit XML
        set +e  # Don't exit on test failures

        # Determine docker runtime flags and pytest command based on dry_run mode
        if [[ "${{ inputs.dry_run }}" == "true" ]]; then
          echo "🔍 Running pytest in dry-run mode (collect-only, no GPU required)"
          GPU_FLAGS=""
          PYTEST_CMD="pytest -v --collect-only -m \"${{ inputs.pytest_marks }}\""
        else
          echo "🚀 Running pytest in normal mode"
          PYTEST_CMD="pytest --continue-on-collection-errors -v --tb=short --basetemp=/tmp/pytest_temp -o cache_dir=/tmp/.pytest_cache --junitxml=/workspace/test-results/${{ env.PYTEST_XML_FILE }} --durations=20 -m \"${{ inputs.pytest_marks }}\""

          # Detect GPU availability and conditionally add GPU flags
          GPU_FLAGS=""
          if command -v nvidia-smi &> /dev/null && nvidia-smi &> /dev/null; then
            echo "✓ GPU detected, enabling GPU runtime"
            GPU_FLAGS="--runtime=nvidia --gpus all"
          else
            echo "⚠️  No GPU detected, running in CPU-only mode"
          fi
        fi

        # Get absolute path for test-results directory and ensure it has proper permissions
        TEST_RESULTS_DIR="$(pwd)/test-results"
        chmod 777 "${TEST_RESULTS_DIR}"
        echo "📁 Test results will be saved to: ${TEST_RESULTS_DIR}"

        DOCKER_ENV_FLAGS=()
        if [[ -n "${HF_TOKEN:-}" ]]; then
          DOCKER_ENV_FLAGS+=(--env "HF_TOKEN=${HF_TOKEN}")
        fi

        docker run ${GPU_FLAGS} --rm -w /workspace \
          --cpus=${NUM_CPUS} \
          --shm-size=200m \
          --network host \
          "${DOCKER_ENV_FLAGS[@]}" \
          --name ${{ env.CONTAINER_ID }}_pytest \
          -v "${TEST_RESULTS_DIR}:/workspace/test-results" \
          ${{ inputs.image_tag }} \
          sh -c "umask 0022 && ${PYTEST_CMD}"

        TEST_EXIT_CODE=$?
        echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> $GITHUB_ENV
        echo "🧪 Tests completed with exit code: ${TEST_EXIT_CODE}"

        # Verify test results were written (only in normal mode)
        if [[ "${{ inputs.dry_run }}" != "true" ]]; then
          if [[ -f "${TEST_RESULTS_DIR}/${{ env.PYTEST_XML_FILE }}" ]]; then
            echo "✅ Test results file found: ${TEST_RESULTS_DIR}/${{ env.PYTEST_XML_FILE }}"
          else
            echo "⚠️  Test results file not found: ${TEST_RESULTS_DIR}/${{ env.PYTEST_XML_FILE }}"
          fi
        fi

        # Always continue to results processing
        exit 0

    - name: Run tests in dind as sidecar mode
      if: inputs.dind_as_sidecar == 'true'
      shell: bash
      env:
        CONTAINER_ID: test_${{ github.run_id }}_${{ github.run_attempt }}_${{ github.job }}
        PYTEST_XML_FILE: pytest_test_report.xml
        HF_HOME: /runner/_work/_temp
        HF_TOKEN: ${{ inputs.hf_token }}
      run: |
        # Run pytest with detailed output and JUnit XML
        set +e  # Don't exit on test failures

        # /dev/shm sizing: NIXL uses UCX which allocates shared memory segments in /dev/shm
        # via shm_open(). Each NIXL agent with num_threads=8 creates 9 UCX workers, each needing
        # ~4.8MB of shm. Disaggregated tests use up to 3 agents (encode+prefill+decode) = ~130MB.
        # Docker's default is 64MB, which is insufficient and causes segfaults.
        # Do NOT use --ipc=host here — it overrides --shm-size with the host's /dev/shm
        # (64MB in K8s pods by default), silently ignoring the size we set.
        DOCKER_OPTS="--shm-size=200m --ulimit memlock=-1 --ulimit stack=67108864"

        # Determine docker runtime flags and pytest command based on dry_run mode
        if [[ "${{ inputs.dry_run }}" == "true" ]]; then
          echo "🔍 Running pytest in dry-run mode (collect-only, no GPU required)"
          GPU_FLAGS=""
          PYTEST_CMD="pytest -v --collect-only -m \"${{ inputs.pytest_marks }}\""
        else
          echo "🚀 Running pytest in normal mode"
          # Detect GPU availability and conditionally add GPU flags
          GPU_FLAGS=""
          # We check 'docker info' for the 'nvidia' runtime, which indicates the Daemon can spawn GPU containers.
          if docker info 2>/dev/null | grep -i "runtimes" | grep -q "nvidia"; then
            echo "✓ Docker Daemon supports Nvidia runtime, enabling GPU flags"
            GPU_FLAGS="--gpus all"
          else
            echo "⚠️  Nvidia runtime not found in Docker Daemon, running in CPU-only mode"
          fi

          # Determine parallelization based on parallel_mode input
          case "${{ inputs.parallel_mode }}" in
            "auto")
              PARALLEL_OPTS="-n auto"
              echo "📊 Parallelization: auto (use all available cores)"
              ;;
            "none"|"0")
              PARALLEL_OPTS="-n 0"
              echo "📊 Parallelization: disabled (sequential execution) for GPU runs"
              ;;
            *)
              PARALLEL_OPTS="-n ${{ inputs.parallel_mode }}"
              echo "📊 Parallelization: ${{ inputs.parallel_mode }} workers"
              ;;
          esac

          # Construct final command with xdist parallelization (-n) and other options
          # --dist=loadscope groups tests by module/class to prevent race conditions in stateful tests
          PYTEST_CMD="pytest ${PARALLEL_OPTS} --dist=loadscope --continue-on-collection-errors -v --tb=short --basetemp=/tmp/pytest_temp -o cache_dir=/tmp/.pytest_cache --junitxml=/workspace/test-results/${{ env.PYTEST_XML_FILE }} --durations=10 -m \"${{ inputs.pytest_marks }}\""
        fi

        # Get absolute path for test-results directory and ensure it has proper permissions
        TEST_RESULTS_DIR="$(pwd)/test-results"
        chmod 777 "${TEST_RESULTS_DIR}"
        echo "📁 Test results will be saved to: ${TEST_RESULTS_DIR}"
        echo "▶️ Executing: $PYTEST_CMD"

        DOCKER_ENV_FLAGS=()
        if [[ -n "${HF_TOKEN:-}" ]]; then
          DOCKER_ENV_FLAGS+=(--env "HF_TOKEN=${HF_TOKEN}")
        fi

        docker run ${GPU_FLAGS} ${DOCKER_OPTS} --rm -w /workspace \
          --network host \
          "${DOCKER_ENV_FLAGS[@]}" \
          --name ${{ env.CONTAINER_ID }}_pytest \
          -v "${TEST_RESULTS_DIR}:/workspace/test-results" \
          ${{ inputs.image_tag }} \
          sh -c "umask 0022 && ${PYTEST_CMD}"

        TEST_EXIT_CODE=$?
        echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> $GITHUB_ENV
        echo "🧪 Tests completed with exit code: ${TEST_EXIT_CODE}"

        # Verify test results were written (only in normal mode)
        if [[ "${{ inputs.dry_run }}" != "true" ]]; then
          if [[ -f "${TEST_RESULTS_DIR}/${{ env.PYTEST_XML_FILE }}" ]]; then
            echo "✅ Test results file found: ${TEST_RESULTS_DIR}/${{ env.PYTEST_XML_FILE }}"
          else
            echo "⚠️  Test results file not found: ${TEST_RESULTS_DIR}/${{ env.PYTEST_XML_FILE }}"
          fi
        fi

        # Always continue to results processing
        exit 0

    - name: Process Test Results
      shell: bash
      run: |

        # Sanitize test_type for filenames (always set this for artifact upload)
        # Remove commas and spaces from test_type for use in filenames
        STR_TEST_TYPE=$(echo "${{ inputs.test_type }}" | tr ', ' '_')
        echo "STR_TEST_TYPE=${STR_TEST_TYPE}" >> $GITHUB_ENV

        # Check for JUnit XML file and determine test status
        JUNIT_FILE="test-results/pytest_test_report.xml"

        if [[ -f "$JUNIT_FILE" ]]; then
          echo "✅ JUnit XML generated successfully"
          # Extract basic test counts for status determination
          TOTAL_TESTS=$(grep -o 'tests="[0-9]*"' "$JUNIT_FILE" | grep -o '[0-9]*' | head -1 || echo "0")
          FAILED_TESTS=$(grep -o 'failures="[0-9]*"' "$JUNIT_FILE" | grep -o '[0-9]*' | head -1 || echo "0")
          ERROR_TESTS=$(grep -o 'errors="[0-9]*"' "$JUNIT_FILE" | grep -o '[0-9]*' | head -1 || echo "0")
          echo "📊 ${TOTAL_TESTS} tests completed (${FAILED_TESTS} failed, ${ERROR_TESTS} errors)"

          # Rename XML file to unique name
          JUNIT_NAME="pytest_test_report_${{ inputs.framework }}_${STR_TEST_TYPE}_${{ inputs.platform_arch }}_${{ github.run_id }}_${{ job.check_run_id }}.xml"
          mv "$JUNIT_FILE" "test-results/$JUNIT_NAME"
          echo "📝 Renamed XML file to: $JUNIT_NAME"
        else
          echo "⚠️  JUnit XML file not found - test results may not be available for upload"
          TOTAL_TESTS=0
          FAILED_TESTS=1  # Treat missing XML as failure
          ERROR_TESTS=0
        fi

        exit ${TEST_EXIT_CODE}

    - name: Cleanup MinIO Service
      if: always() && inputs.start_minio == 'true'
      shell: bash
      run: |
        echo "🧹 Cleaning up MinIO container..."
        docker rm -f dynamo-minio-test 2>/dev/null || true

    - name: Upload Test Results
      uses: actions/upload-artifact@v4
      if: always()  # Always upload test results, even if tests failed
      with:
        name: test-results-${{ inputs.framework }}-${{ env.STR_TEST_TYPE }}-${{ env.PLATFORM_ARCH }}-${{ github.run_id }}-${{ job.check_run_id }}
        path: test-results/pytest_test_report_${{ inputs.framework }}_${{ env.STR_TEST_TYPE }}_${{ inputs.platform_arch }}_${{ github.run_id }}_${{ job.check_run_id }}.xml
        retention-days: 7
