[Bugfix][Harmless] Fix float16 dtype for model_is_embedding (#7566)

9c8e2d11 · Michael Goin · GitHub · 21313e09 · 9c8e2d11
Unverified Commit 9c8e2d11 authored Aug 15, 2024 by Michael Goin Committed by GitHub Aug 15, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

vllm/entrypoints/openai/api_server.py vllm/entrypoints/openai/api_server.py +1 -1

No files found.
--- a/vllm/entrypoints/openai/api_server.py
+++ b/vllm/entrypoints/openai/api_server.py
@@ -66,7 +66,7 @@ def model_is_embedding(model_name: str, trust_remote_code: bool) -> bool:
                       tokenizer_mode="auto",
                       trust_remote_code=trust_remote_code,
                       seed=0,
-                       dtype="float16").embedding_mode
+                       dtype="auto").embedding_mode


 @asynccontextmanager