Unverified Commit 0bd2a573 authored by Michele Catalano's avatar Michele Catalano Committed by GitHub
Browse files

Allow send list of str for the Prompt on openai demo endpoint /v1/completions (#323)



* allow str or List[str] for prompt

* Update vllm/entrypoints/openai/api_server.py
Co-authored-by: default avatarZhuohan Li <zhuohan123@gmail.com>

---------
Co-authored-by: default avatarZhuohan Li <zhuohan123@gmail.com>
parent 49b26e2c
...@@ -357,7 +357,11 @@ async def create_completion(raw_request: Request): ...@@ -357,7 +357,11 @@ async def create_completion(raw_request: Request):
model_name = request.model model_name = request.model
request_id = f"cmpl-{random_uuid()}" request_id = f"cmpl-{random_uuid()}"
prompt = request.prompt if isinstance(request.prompt, list):
assert len(request.prompt) == 1
prompt = request.prompt[0]
else:
prompt = request.prompt
created_time = int(time.time()) created_time = int(time.time())
try: try:
sampling_params = SamplingParams( sampling_params = SamplingParams(
......
...@@ -73,7 +73,7 @@ class ChatCompletionRequest(BaseModel): ...@@ -73,7 +73,7 @@ class ChatCompletionRequest(BaseModel):
class CompletionRequest(BaseModel): class CompletionRequest(BaseModel):
model: str model: str
prompt: str prompt: Union[str, List[str]]
suffix: Optional[str] = None suffix: Optional[str] = None
max_tokens: Optional[int] = 16 max_tokens: Optional[int] = 16
temperature: Optional[float] = 1.0 temperature: Optional[float] = 1.0
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment