Unverified Commit 40facad5 authored by Toshiki Kataoka's avatar Toshiki Kataoka Committed by GitHub
Browse files

feat: support token ids in /v1/completions (#736)

parent da504445
......@@ -94,9 +94,14 @@ def load_chat_template_for_openai_api(chat_template_arg):
async def v1_completions(tokenizer_manager, raw_request: Request):
request_json = await raw_request.json()
request = CompletionRequest(**request_json)
prompt = request.prompt
if isinstance(prompt, str) or isinstance(prompt[0], str):
prompt_kwargs = {"text": prompt}
else:
prompt_kwargs = {"input_ids": prompt}
adapted_request = GenerateReqInput(
text=request.prompt,
**prompt_kwargs,
sampling_params={
"temperature": request.temperature,
"max_new_tokens": request.max_tokens,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment