[Bugfix] Include encoder prompts len to non-stream api usage response (#8861)

93d364da · Pernekhan Utemuratov · GitHub · d9cfbc89 · 93d364da
Unverified Commit 93d364da authored Sep 26, 2024 by Pernekhan Utemuratov Committed by GitHub Sep 26, 2024
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 0 deletions

vllm/entrypoints/openai/serving_chat.py vllm/entrypoints/openai/serving_chat.py +2 -0

No files found.
--- a/vllm/entrypoints/openai/serving_chat.py
+++ b/vllm/entrypoints/openai/serving_chat.py
@@ -726,6 +726,8 @@ class OpenAIServingChat(OpenAIServing):

        assert final_res.prompt_token_ids is not None
        num_prompt_tokens = len(final_res.prompt_token_ids)
+        if final_res.encoder_prompt_token_ids is not None:
+            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
        usage = UsageInfo(