ci: update `lmms-eval` to speed up multimodal CI (#11000)

scripts/ci/amd_ci_install_dependency.sh

@@ -22,7 +22,7 @@ case "${GPU_ARCH}" in
     docker exec ci_sglang rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml
     docker exec ci_sglang pip install -e "python[dev_hip]" --no-deps # TODO: only for mi35x
     # For lmms_evals evaluating MMMU
-    docker exec -w / ci_sglang git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+    docker exec -w / ci_sglang git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
     docker exec -w /lmms-eval ci_sglang pip install -e . --no-deps # TODO: only for mi35x
     ;;
   mi30x|mi300|mi325)
@@ -30,7 +30,7 @@ case "${GPU_ARCH}" in
     docker exec ci_sglang rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml
     docker exec ci_sglang pip install -e "python[dev_hip]"
     # For lmms_evals evaluating MMMU
-    docker exec -w / ci_sglang git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+    docker exec -w / ci_sglang git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
     docker exec -w /lmms-eval ci_sglang pip install -e .
     ;;
   *)

scripts/ci/ci_install_dependency.sh

@@ -66,7 +66,7 @@ $PIP_CMD install mooncake-transfer-engine==0.3.6.post1 nvidia-cuda-nvrtc-cu12 py
 
 if [ "$IS_BLACKWELL" != "1" ]; then
     # For lmms_evals evaluating MMMU
-    git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+    git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
     $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX
 
     # Install xformers

test/srt/models/test_vlm_models.py

@@ -81,7 +81,7 @@ class TestVLMModels(CustomTestCase):
         model = "openai_compatible"
         tp = 1
         tasks = "mmmu_val"
-        batch_size = 2
+        batch_size = 32
         log_suffix = "openai_compatible"
         os.makedirs(output_path, exist_ok=True)
 
@@ -179,7 +179,15 @@ class TestVLMModels(CustomTestCase):
             self.run_mmmu_eval(model.model, output_path)
 
             # Get the result file
-            result_file_path = glob.glob(f"{output_path}/*.json")[0]
+            # Search recursively for JSON result files (lmms-eval v0.4.1+ creates subdirectories)
+            result_files = glob.glob(f"{output_path}/**/*.json", recursive=True)
+            if not result_files:
+                result_files = glob.glob(f"{output_path}/*.json")
+
+            if not result_files:
+                raise FileNotFoundError(f"No JSON result files found in {output_path}")
+
+            result_file_path = result_files[0]
 
             with open(result_file_path, "r") as f:
                 result = json.load(f)