Skip to content

GitLab

Unverified Commit 5e0391c0 authored by Alex Wu's avatar Alex Wu Committed by GitHub
Browse files

[Frontend] Separate OpenAI Batch Runner usage from API Server (#4851)

parent dbc0754d
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 1 deletion
vllm/entrypoints/openai/run_batch.py
View file @ 5e0391c0
...@@ -101,7 +101,7 @@ async def main(args): ...@@ -101,7 +101,7 @@ async def main(args):
engine_args = AsyncEngineArgs.from_cli_args(args) engine_args = AsyncEngineArgs.from_cli_args(args)
engine = AsyncLLMEngine.from_engine_args( engine = AsyncLLMEngine.from_engine_args(
engine_args, usage_context=UsageContext.OPENAI_API_SERVER) engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
# When using single vLLM without engine_use_ray # When using single vLLM without engine_use_ray
model_config = await engine.get_model_config() model_config = await engine.get_model_config()
......
vllm/usage/usage_lib.py
View file @ 5e0391c0
...@@ -90,6 +90,7 @@ class UsageContext(str, Enum): ...@@ -90,6 +90,7 @@ class UsageContext(str, Enum):
LLM_CLASS = "LLM_CLASS" LLM_CLASS = "LLM_CLASS"
API_SERVER = "API_SERVER" API_SERVER = "API_SERVER"
OPENAI_API_SERVER = "OPENAI_API_SERVER" OPENAI_API_SERVER = "OPENAI_API_SERVER"
OPENAI_BATCH_RUNNER = "OPENAI_BATCH_RUNNER"
ENGINE_CONTEXT = "ENGINE_CONTEXT" ENGINE_CONTEXT = "ENGINE_CONTEXT"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment