Unverified Commit 40facad5 authored by Toshiki Kataoka's avatar Toshiki Kataoka Committed by GitHub
Browse files

feat: support token ids in /v1/completions (#736)

parent da504445
...@@ -94,9 +94,14 @@ def load_chat_template_for_openai_api(chat_template_arg): ...@@ -94,9 +94,14 @@ def load_chat_template_for_openai_api(chat_template_arg):
async def v1_completions(tokenizer_manager, raw_request: Request): async def v1_completions(tokenizer_manager, raw_request: Request):
request_json = await raw_request.json() request_json = await raw_request.json()
request = CompletionRequest(**request_json) request = CompletionRequest(**request_json)
prompt = request.prompt
if isinstance(prompt, str) or isinstance(prompt[0], str):
prompt_kwargs = {"text": prompt}
else:
prompt_kwargs = {"input_ids": prompt}
adapted_request = GenerateReqInput( adapted_request = GenerateReqInput(
text=request.prompt, **prompt_kwargs,
sampling_params={ sampling_params={
"temperature": request.temperature, "temperature": request.temperature,
"max_new_tokens": request.max_tokens, "max_new_tokens": request.max_tokens,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment