Unverified Commit 5e11b403 authored by Alex Brooks's avatar Alex Brooks Committed by GitHub
Browse files

[Frontend] Delegate to vLLM Omni When `--omni` Passed (#40744)


Signed-off-by: default avatarAlex Brooks <albrooks@redhat.com>
parent f768b447
......@@ -7,6 +7,7 @@ to avoid certain eager import breakage."""
import importlib.metadata
import sys
from importlib.util import find_spec
from vllm.logger import init_logger
......@@ -34,6 +35,22 @@ def main():
cli_env_setup()
# If `--omni` arg is passed to the CLI, delegate to vLLM Omni's entrypoint handling
if "--omni" in sys.argv:
# NOTE: Check the spec instead of importing directly here, since things could
# fail with ImportError due to mismatched versions if things are moved around.
spec = find_spec("vllm_omni")
if spec is None:
logger.error(
"--omni flag requires a valid instance of vllm-omni to be installed."
)
sys.exit(1)
from vllm_omni.entrypoints.cli.main import main as omni_main
logger.info("Delegating entrypoint handling to vllm-omni")
omni_main()
else:
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
if len(sys.argv) > 1 and sys.argv[1] == "bench":
logger.debug(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment