Unverified Commit 1f70313e authored by Andreas Karatzas's avatar Andreas Karatzas Committed by GitHub
Browse files

[Bugfix] Fix ScoreMultiModalParam multi-document scoring returning single result (#33837)


Signed-off-by: default avatarAndreas Karatzas <akaratza@amd.com>
Signed-off-by: default avatarwang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: default avatarwang.yuqi <yuqi.wang@daocloud.io>
parent 07daee13
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import pytest import pytest
import transformers import transformers
...@@ -117,7 +116,7 @@ def _normalize_image(image_val: str) -> str: ...@@ -117,7 +116,7 @@ def _normalize_image(image_val: str) -> str:
def create_score_multimodal_param( def create_score_multimodal_param(
content_parts: list[dict], content_parts: list[dict],
) -> ScoreMultiModalParam: ) -> list[ScoreMultiModalParam]:
""" """
Create a ScoreMultiModalParam from a list of content dictionaries. Create a ScoreMultiModalParam from a list of content dictionaries.
...@@ -152,7 +151,7 @@ def create_score_multimodal_param( ...@@ -152,7 +151,7 @@ def create_score_multimodal_param(
) )
) )
return ScoreMultiModalParam(content=formatted_content) return [ScoreMultiModalParam(content=[content]) for content in formatted_content]
def _run_vllm( def _run_vllm(
...@@ -198,23 +197,7 @@ def _run_hf( ...@@ -198,23 +197,7 @@ def _run_hf(
else: else:
raise ValueError("Unsupported query format") raise ValueError("Unsupported query format")
# Separate documents by type scores: list[float] = []
text_docs: list[str] = []
image_docs: list[str] = []
text_indices: list[int] = []
image_indices: list[int] = []
for idx, doc in enumerate(document_strs):
if "text" in doc:
text_docs.append(doc["text"])
text_indices.append(idx)
elif "image" in doc:
image_docs.append(_normalize_image(doc["image"]))
image_indices.append(idx)
else:
raise ValueError(f"Unsupported document format at index {idx}")
scores: list[None | float] = [None] * len(document_strs)
with hf_runner( with hf_runner(
model, model,
...@@ -223,30 +206,24 @@ def _run_hf( ...@@ -223,30 +206,24 @@ def _run_hf(
auto_cls=AutoModel, auto_cls=AutoModel,
model_kwargs={"key_mapping": CHECKPOINT_TO_HF_MAPPER}, model_kwargs={"key_mapping": CHECKPOINT_TO_HF_MAPPER},
) as hf_model: ) as hf_model:
# Score text documents for doc in document_strs:
if text_docs: if "text" in doc:
text_scores = hf_model.model.compute_score( score = hf_model.model.compute_score(
[[query_data, d] for d in text_docs], [[query_data, doc["text"]]],
max_length=2048, max_length=2048,
query_type=query_type, query_type=query_type,
doc_type="text", doc_type="text",
) )
for i, s in zip(text_indices, text_scores): scores.append(score)
scores[i] = s elif "image" in doc:
score = hf_model.model.compute_score(
# Score image documents [[query_data, doc["image"]]],
if image_docs: max_length=2048,
image_scores = hf_model.model.compute_score( query_type=query_type,
[[query_data, d] for d in image_docs], doc_type="image",
max_length=2048, )
query_type=query_type, scores.append(score)
doc_type="image", return scores
)
for i, s in zip(image_indices, image_scores):
scores[i] = s
assert all(s is not None for s in scores)
return cast(list[float], scores)
def _run_test( def _run_test(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment