adapter.py 49 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
"""
Copyright 2023-2024 SGLang Team
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

16
"""Conversion between OpenAI APIs and native SRT APIs"""
Liangsheng Yin's avatar
Liangsheng Yin committed
17

18
import asyncio
19
import json
20
import logging
21
import os
22
23
import time
import uuid
24
from http import HTTPStatus
25
from typing import Dict, List, Optional
26

27
from fastapi import HTTPException, Request, UploadFile
28
from fastapi.responses import JSONResponse, StreamingResponse
29
from pydantic import ValidationError
30
31
32
33
34
35
36
37

from sglang.srt.conversation import (
    Conversation,
    SeparatorStyle,
    chat_template_exists,
    generate_chat_conv,
    register_conv_template,
)
Ying Sheng's avatar
Ying Sheng committed
38
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
Mingyi's avatar
Mingyi committed
39
from sglang.srt.openai_api.protocol import (
40
41
    BatchRequest,
    BatchResponse,
42
43
44
45
46
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
47
    ChatCompletionTokenLogprob,
48
    ChatMessage,
49
    ChoiceLogprobs,
50
51
52
53
54
55
    CompletionRequest,
    CompletionResponse,
    CompletionResponseChoice,
    CompletionResponseStreamChoice,
    CompletionStreamResponse,
    DeltaMessage,
Ying Sheng's avatar
Ying Sheng committed
56
    EmbeddingObject,
57
58
    EmbeddingRequest,
    EmbeddingResponse,
59
    ErrorResponse,
60
    FileDeleteResponse,
61
62
    FileRequest,
    FileResponse,
63
    LogProbs,
64
    TopLogprob,
65
66
67
    UsageInfo,
)

68
69
logger = logging.getLogger(__name__)

70
71
chat_template_name = None

Liangsheng Yin's avatar
Liangsheng Yin committed
72

73
74
75
76
77
78
79
80
81
82
class FileMetadata:
    def __init__(self, filename: str, purpose: str):
        self.filename = filename
        self.purpose = purpose


# In-memory storage for batch jobs and files
batch_storage: Dict[str, BatchResponse] = {}
file_id_request: Dict[str, FileMetadata] = {}
file_id_response: Dict[str, FileResponse] = {}
83
# map file id to file path in SGLang backend
84
85
86
87
88
89
90
file_id_storage: Dict[str, str] = {}


# backend storage directory
storage_dir = None


91
92
93
94
95
96
97
98
99
100
101
102
103
def format_finish_reason(finish_reason) -> Optional[str]:
    if finish_reason.startswith("None"):
        return None
    elif finish_reason.startswith("FINISH_MATCHED"):
        return "stop"
    elif finish_reason.startswith("FINISH_LENGTH"):
        return "length"
    elif finish_reason.startswith("FINISH_ABORT"):
        return "abort"
    else:
        return "unknown"


104
105
106
def create_error_response(
    message: str,
    err_type: str = "BadRequestError",
107
108
109
110
    status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
):
    error = ErrorResponse(message=message, type=err_type, code=status_code.value)
    return JSONResponse(content=error.model_dump(), status_code=error.code)
111
112
113
114
115


def create_streaming_error_response(
    message: str,
    err_type: str = "BadRequestError",
116
117
118
    status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
) -> str:
    error = ErrorResponse(message=message, type=err_type, code=status_code.value)
119
120
121
122
    json_str = json.dumps({"error": error.model_dump()})
    return json_str


123
def load_chat_template_for_openai_api(tokenizer_manager, chat_template_arg):
124
125
    global chat_template_name

126
    logger.info(f"Use chat template: {chat_template_arg}")
127
128
129
130
131
132
    if not chat_template_exists(chat_template_arg):
        if not os.path.exists(chat_template_arg):
            raise RuntimeError(
                f"Chat template {chat_template_arg} is not a built-in template name "
                "or a valid chat template file path."
            )
133
134
135
136
137
        if chat_template_arg.endswith(".jinja"):
            with open(chat_template_arg, "r") as f:
                chat_template = "".join(f.readlines()).strip("\n")
            tokenizer_manager.tokenizer.chat_template = chat_template.replace(
                "\\n", "\n"
138
            )
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
            chat_template_name = None
        else:
            assert chat_template_arg.endswith(
                ".json"
            ), "unrecognized format of chat template file"
            with open(chat_template_arg, "r") as filep:
                template = json.load(filep)
                try:
                    sep_style = SeparatorStyle[template["sep_style"]]
                except KeyError:
                    raise ValueError(
                        f"Unknown separator style: {template['sep_style']}"
                    ) from None
                register_conv_template(
                    Conversation(
                        name=template["name"],
                        system_template=template["system"] + "\n{system_message}",
                        system_message=template.get("system_message", ""),
                        roles=(template["user"], template["assistant"]),
                        sep_style=sep_style,
                        sep=template.get("sep", "\n"),
                        stop_str=template["stop_str"],
                    ),
                    override=True,
                )
            chat_template_name = template["name"]
165
166
167
168
    else:
        chat_template_name = chat_template_arg


169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
async def v1_files_create(file: UploadFile, purpose: str, file_storage_pth: str = None):
    try:
        global storage_dir
        if file_storage_pth:
            storage_dir = file_storage_pth
        # Read the file content
        file_content = await file.read()

        # Create an instance of RequestBody
        request_body = FileRequest(file=file_content, purpose=purpose)

        # Save the file to the sglang_oai_storage directory
        os.makedirs(storage_dir, exist_ok=True)
        file_id = f"backend_input_file-{uuid.uuid4()}"
        filename = f"{file_id}.jsonl"
        file_path = os.path.join(storage_dir, filename)

        with open(file_path, "wb") as f:
            f.write(request_body.file)

        # add info to global file map
        file_id_request[file_id] = FileMetadata(filename=file.filename, purpose=purpose)
        file_id_storage[file_id] = file_path

        # Return the response in the required format
        response = FileResponse(
            id=file_id,
            bytes=len(request_body.file),
            created_at=int(time.time()),
            filename=file.filename,
            purpose=request_body.purpose,
        )
        file_id_response[file_id] = response

        return response
    except ValidationError as e:
        return {"error": "Invalid input", "details": e.errors()}


208
209
210
211
212
213
214
215
216
217
218
219
220
221
async def v1_delete_file(file_id: str):
    # Retrieve the file job from the in-memory storage
    file_response = file_id_response.get(file_id)
    if file_response is None:
        raise HTTPException(status_code=404, detail="File not found")
    file_path = file_id_storage.get(file_id)
    if file_path is None:
        raise HTTPException(status_code=404, detail="File not found")
    os.remove(file_path)
    del file_id_response[file_id]
    del file_id_storage[file_id]
    return FileDeleteResponse(id=file_id, deleted=True)


222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
async def v1_batches(tokenizer_manager, raw_request: Request):
    try:
        body = await raw_request.json()

        batch_request = BatchRequest(**body)

        batch_id = f"batch_{uuid.uuid4()}"

        # Create an instance of BatchResponse
        batch_response = BatchResponse(
            id=batch_id,
            endpoint=batch_request.endpoint,
            input_file_id=batch_request.input_file_id,
            completion_window=batch_request.completion_window,
            created_at=int(time.time()),
            metadata=batch_request.metadata,
        )

        batch_storage[batch_id] = batch_response

        # Start processing the batch asynchronously
        asyncio.create_task(process_batch(tokenizer_manager, batch_id, batch_request))

        # Return the initial batch_response
        return batch_response

    except ValidationError as e:
        return {"error": "Invalid input", "details": e.errors()}
    except Exception as e:
        return {"error": str(e)}


async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRequest):
    try:
        # Update the batch status to "in_progress"
        batch_storage[batch_id].status = "in_progress"
        batch_storage[batch_id].in_progress_at = int(time.time())

        # Retrieve the input file content
        input_file_request = file_id_request.get(batch_request.input_file_id)
        if not input_file_request:
            raise ValueError("Input file not found")

        # Parse the JSONL file and process each request
        input_file_path = file_id_storage.get(batch_request.input_file_id)
        with open(input_file_path, "r", encoding="utf-8") as f:
            lines = f.readlines()

        total_requests = len(lines)
        completed_requests = 0
        failed_requests = 0

        all_ret = []
        end_point = batch_storage[batch_id].endpoint
        file_request_list = []
        all_requests = []
278
        request_ids = []
279
280
281
282
        for line in lines:
            request_data = json.loads(line)
            file_request_list.append(request_data)
            body = request_data["body"]
283
            request_ids.append(request_data["custom_id"])
284
285
286
287
288
289

            # Although streaming is supported for standalone completions, it is not supported in
            # batch mode (multiple completions in single request).
            if body.get("stream", False):
                raise ValueError("Streaming requests are not supported in batch mode")

290
291
292
293
            if end_point == "/v1/chat/completions":
                all_requests.append(ChatCompletionRequest(**body))
            elif end_point == "/v1/completions":
                all_requests.append(CompletionRequest(**body))
294

295
296
        if end_point == "/v1/chat/completions":
            adapted_request, request = v1_chat_generate_request(
297
                all_requests, tokenizer_manager, request_ids=request_ids
298
299
            )
        elif end_point == "/v1/completions":
300
301
302
303
            adapted_request, request = v1_generate_request(
                all_requests, request_ids=request_ids
            )

304
305
306
307
308
309
310
        try:
            ret = await tokenizer_manager.generate_request(adapted_request).__anext__()
            if not isinstance(ret, list):
                ret = [ret]
            if end_point == "/v1/chat/completions":
                responses = v1_chat_generate_response(request, ret, to_file=True)
            else:
yichuan~'s avatar
yichuan~ committed
311
312
313
                responses = v1_generate_response(
                    request, ret, tokenizer_manager, to_file=True
                )
314
315
316
317
318
319
320
321
322
323
324
325

        except Exception as e:
            error_json = {
                "id": f"batch_req_{uuid.uuid4()}",
                "custom_id": request_data.get("custom_id"),
                "response": None,
                "error": {"message": str(e)},
            }
            all_ret.append(error_json)
            failed_requests += len(file_request_list)

        for idx, response in enumerate(responses):
326
            # the batch_req here can be changed to be named within a batch granularity
327
328
329
330
331
332
333
334
            response_json = {
                "id": f"batch_req_{uuid.uuid4()}",
                "custom_id": file_request_list[idx].get("custom_id"),
                "response": response,
                "error": None,
            }
            all_ret.append(response_json)
            completed_requests += 1
335

336
337
338
339
340
341
342
343
344
345
346
347
        # Write results to a new file
        output_file_id = f"backend_result_file-{uuid.uuid4()}"
        global storage_dir
        output_file_path = os.path.join(storage_dir, f"{output_file_id}.jsonl")
        with open(output_file_path, "w", encoding="utf-8") as f:
            for ret in all_ret:
                f.write(json.dumps(ret) + "\n")

        # Update batch response with output file information
        retrieve_batch = batch_storage[batch_id]
        retrieve_batch.output_file_id = output_file_id
        file_id_storage[output_file_id] = output_file_path
348
349
350
351
352
353
354
        file_id_response[output_file_id] = FileResponse(
            id=output_file_id,
            bytes=os.path.getsize(output_file_path),
            created_at=int(time.time()),
            filename=f"{output_file_id}.jsonl",
            purpose="batch_result",
        )
355
356
357
358
359
360
361
362
363
364
        # Update batch status to "completed"
        retrieve_batch.status = "completed"
        retrieve_batch.completed_at = int(time.time())
        retrieve_batch.request_counts = {
            "total": total_requests,
            "completed": completed_requests,
            "failed": failed_requests,
        }

    except Exception as e:
365
        logger.error("error in SGLang:", e)
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
        # Update batch status to "failed"
        retrieve_batch = batch_storage[batch_id]
        retrieve_batch.status = "failed"
        retrieve_batch.failed_at = int(time.time())
        retrieve_batch.errors = {"message": str(e)}


async def v1_retrieve_batch(batch_id: str):
    # Retrieve the batch job from the in-memory storage
    batch_response = batch_storage.get(batch_id)
    if batch_response is None:
        raise HTTPException(status_code=404, detail="Batch not found")

    return batch_response


382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
async def v1_cancel_batch(tokenizer_manager, batch_id: str):
    # Retrieve the batch job from the in-memory storage
    batch_response = batch_storage.get(batch_id)
    if batch_response is None:
        raise HTTPException(status_code=404, detail="Batch not found")

    # Only do cancal when status is "validating" or "in_progress"
    if batch_response.status in ["validating", "in_progress"]:
        # Start cancelling the batch asynchronously
        asyncio.create_task(
            cancel_batch(
                tokenizer_manager=tokenizer_manager,
                batch_id=batch_id,
                input_file_id=batch_response.input_file_id,
            )
        )

        # Update batch status to "cancelling"
        batch_response.status = "cancelling"

        return batch_response
    else:
        raise HTTPException(
            status_code=500,
            detail=f"Current status is {batch_response.status}, no need to cancel",
        )


async def cancel_batch(tokenizer_manager, batch_id: str, input_file_id: str):
    try:
        # Update the batch status to "cancelling"
        batch_storage[batch_id].status = "cancelling"

        # Retrieve the input file content
        input_file_request = file_id_request.get(input_file_id)
        if not input_file_request:
            raise ValueError("Input file not found")

        # Parse the JSONL file and process each request
        input_file_path = file_id_storage.get(input_file_id)
        with open(input_file_path, "r", encoding="utf-8") as f:
            lines = f.readlines()

        file_request_list = []
        request_ids = []
        for line in lines:
            request_data = json.loads(line)
            file_request_list.append(request_data)
            request_ids.append(request_data["custom_id"])

        # Cancel requests by request_ids
        for rid in request_ids:
            tokenizer_manager.abort_request(rid=rid)

        retrieve_batch = batch_storage[batch_id]
        retrieve_batch.status = "cancelled"

    except Exception as e:
        logger.error("error in SGLang:", e)
        # Update batch status to "failed"
        retrieve_batch = batch_storage[batch_id]
        retrieve_batch.status = "failed"
        retrieve_batch.failed_at = int(time.time())
        retrieve_batch.errors = {"message": str(e)}


448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
async def v1_retrieve_file(file_id: str):
    # Retrieve the batch job from the in-memory storage
    file_response = file_id_response.get(file_id)
    if file_response is None:
        raise HTTPException(status_code=404, detail="File not found")
    return file_response


async def v1_retrieve_file_content(file_id: str):
    file_pth = file_id_storage.get(file_id)
    if not file_pth or not os.path.exists(file_pth):
        raise HTTPException(status_code=404, detail="File not found")

    def iter_file():
        with open(file_pth, mode="rb") as file_like:
            yield from file_like

    return StreamingResponse(iter_file(), media_type="application/octet-stream")


468
469
470
def v1_generate_request(
    all_requests: List[CompletionRequest], request_ids: List[str] = None
):
471
472
    prompts = []
    sampling_params_list = []
473
    return_logprobs = []
474
    logprob_start_lens = []
475
    top_logprobs_nums = []
yichuan~'s avatar
yichuan~ committed
476

477
478
    # NOTE: with openai API, the prompt's logprobs are always not computed
    first_prompt_type = type(all_requests[0].prompt)
479
480
    for request in all_requests:
        assert (
481
            type(request.prompt) == first_prompt_type
482
        ), "All prompts must be of the same type in file input settings"
483
484
485
486
487
        if len(all_requests) > 1 and request.n > 1:
            raise ValueError(
                "Parallel sampling is not supported for completions from files"
            )
        if request.echo and request.logprobs:
488
            logger.warning(
489
490
491
492
493
494
                "Echo is not compatible with logprobs. "
                "To compute logprobs of input prompt, please use SGLang /request API."
            )

    for request in all_requests:
        prompts.append(request.prompt)
495
        return_logprobs.append(request.logprobs is not None and request.logprobs > 0)
496
        logprob_start_lens.append(-1)
497
498
499
        top_logprobs_nums.append(
            request.logprobs if request.logprobs is not None else 0
        )
500
501
502
503
        sampling_params_list.append(
            {
                "temperature": request.temperature,
                "max_new_tokens": request.max_tokens,
504
                "min_new_tokens": request.min_tokens,
505
                "stop": request.stop,
506
                "stop_token_ids": request.stop_token_ids,
507
508
509
                "top_p": request.top_p,
                "presence_penalty": request.presence_penalty,
                "frequency_penalty": request.frequency_penalty,
510
                "repetition_penalty": request.repetition_penalty,
511
                "regex": request.regex,
512
                "json_schema": request.json_schema,
513
514
515
516
517
518
519
520
                "n": request.n,
                "ignore_eos": request.ignore_eos,
            }
        )

    if len(all_requests) == 1:
        prompt = prompts[0]
        sampling_params_list = sampling_params_list[0]
521
        logprob_start_lens = logprob_start_lens[0]
522
523
        return_logprobs = return_logprobs[0]
        top_logprobs_nums = top_logprobs_nums[0]
yichuan~'s avatar
yichuan~ committed
524
        if isinstance(prompt, str) or isinstance(prompt[0], str):
525
526
527
            prompt_kwargs = {"text": prompt}
        else:
            prompt_kwargs = {"input_ids": prompt}
528
    else:
529
        if isinstance(prompts[0], str):
530
531
532
            prompt_kwargs = {"text": prompts}
        else:
            prompt_kwargs = {"input_ids": prompts}
yichuan~'s avatar
yichuan~ committed
533

534
    adapted_request = GenerateReqInput(
535
        **prompt_kwargs,
536
        sampling_params=sampling_params_list,
537
538
        return_logprob=return_logprobs,
        top_logprobs_num=top_logprobs_nums,
539
        logprob_start_len=logprob_start_lens,
540
        return_text_in_logprobs=True,
541
        stream=all_requests[0].stream,
542
        rid=request_ids,
543
    )
yichuan~'s avatar
yichuan~ committed
544

545
546
547
548
549
    if len(all_requests) == 1:
        return adapted_request, all_requests[0]
    return adapted_request, all_requests


yichuan~'s avatar
yichuan~ committed
550
def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
551
552
553
    choices = []
    echo = False

yichuan~'s avatar
yichuan~ committed
554
    if (not isinstance(request, list)) and request.echo:
555
        # TODO: handle the case propmt is token ids
yichuan~'s avatar
yichuan~ committed
556
557
        if isinstance(request.prompt, list) and isinstance(request.prompt[0], str):
            # for the case of multiple str prompts
558
            prompts = request.prompt
yichuan~'s avatar
yichuan~ committed
559
560
561
562
563
564
565
566
567
568
569
570
571
        elif isinstance(request.prompt, list) and isinstance(request.prompt[0], list):
            # for the case of multiple token ids prompts
            prompts = [
                tokenizer_manager.tokenizer.decode(prompt, skip_special_tokens=True)
                for prompt in request.prompt
            ]
        elif isinstance(request.prompt, list) and isinstance(request.prompt[0], int):
            # for the case of single token ids prompt
            prompts = [
                tokenizer_manager.tokenizer.decode(
                    request.prompt, skip_special_tokens=True
                )
            ]
572
        else:
yichuan~'s avatar
yichuan~ committed
573
            # for the case of single str prompt
574
575
576
577
578
            prompts = [request.prompt]
        echo = True

    for idx, ret_item in enumerate(ret):
        text = ret_item["text"]
yichuan~'s avatar
yichuan~ committed
579
        if isinstance(request, list) and request[idx].echo:
580
581
            echo = True
            text = request[idx].prompt + text
yichuan~'s avatar
yichuan~ committed
582
583
584
        if (not isinstance(request, list)) and echo:
            prompt_index = idx // request.n
            text = prompts[prompt_index] + text
585
586

        logprobs = False
yichuan~'s avatar
yichuan~ committed
587
        if isinstance(request, list) and request[idx].logprobs:
588
            logprobs = True
yichuan~'s avatar
yichuan~ committed
589
        elif (not isinstance(request, list)) and request.logprobs:
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
            logprobs = True
        if logprobs:
            if echo:
                input_token_logprobs = ret_item["meta_info"]["input_token_logprobs"]
                input_top_logprobs = ret_item["meta_info"]["input_top_logprobs"]
            else:
                input_token_logprobs = None
                input_top_logprobs = None

            logprobs = to_openai_style_logprobs(
                input_token_logprobs=input_token_logprobs,
                input_top_logprobs=input_top_logprobs,
                output_token_logprobs=ret_item["meta_info"]["output_token_logprobs"],
                output_top_logprobs=ret_item["meta_info"]["output_top_logprobs"],
            )
        else:
            logprobs = None

        if to_file:
609
            # to make the choise data json serializable
610
611
612
613
            choice_data = {
                "index": 0,
                "text": text,
                "logprobs": logprobs,
614
615
616
                "finish_reason": format_finish_reason(
                    ret_item["meta_info"]["finish_reason"]
                ),
617
618
619
620
621
622
            }
        else:
            choice_data = CompletionResponseChoice(
                index=idx,
                text=text,
                logprobs=logprobs,
623
624
625
                finish_reason=format_finish_reason(
                    ret_item["meta_info"]["finish_reason"]
                ),
626
627
628
629
630
631
632
633
634
635
636
            )

        choices.append(choice_data)

    if to_file:
        responses = []
        for i, choice in enumerate(choices):
            response = {
                "status_code": 200,
                "request_id": ret[i]["meta_info"]["id"],
                "body": {
637
                    # remain the same but if needed we can change that
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
                    "id": ret[i]["meta_info"]["id"],
                    "object": "text_completion",
                    "created": int(time.time()),
                    "model": request[i].model,
                    "choices": choice,
                    "usage": {
                        "prompt_tokens": ret[i]["meta_info"]["prompt_tokens"],
                        "completion_tokens": ret[i]["meta_info"]["completion_tokens"],
                        "total_tokens": ret[i]["meta_info"]["prompt_tokens"]
                        + ret[i]["meta_info"]["completion_tokens"],
                    },
                    "system_fingerprint": None,
                },
            }
            responses.append(response)
        return responses
    else:
655
656
657
        prompt_tokens = sum(
            ret[i]["meta_info"]["prompt_tokens"] for i in range(0, len(ret), request.n)
        )
658
659
660
661
662
663
        completion_tokens = sum(item["meta_info"]["completion_tokens"] for item in ret)
        response = CompletionResponse(
            id=ret[0]["meta_info"]["id"],
            model=request.model,
            choices=choices,
            usage=UsageInfo(
yichuan~'s avatar
yichuan~ committed
664
                prompt_tokens=prompt_tokens,
665
                completion_tokens=completion_tokens,
yichuan~'s avatar
yichuan~ committed
666
                total_tokens=prompt_tokens + completion_tokens,
667
668
669
670
671
672
673
674
675
            ),
        )
    return response


async def v1_completions(tokenizer_manager, raw_request: Request):
    request_json = await raw_request.json()
    all_requests = [CompletionRequest(**request_json)]
    adapted_request, request = v1_generate_request(all_requests)
676
677
678
679

    if adapted_request.stream:

        async def generate_stream_resp():
680
681
682
683
            stream_buffers = {}
            n_prev_tokens = {}
            prompt_tokens = {}
            completion_tokens = {}
684
685
            try:
                async for content in tokenizer_manager.generate_request(
686
687
                    adapted_request, raw_request
                ):
688
689
690
691
692
                    index = content["index"]

                    stream_buffer = stream_buffers.get(index, "")
                    n_prev_token = n_prev_tokens.get(index, 0)

693
                    text = content["text"]
694
695
                    prompt_tokens[index] = content["meta_info"]["prompt_tokens"]
                    completion_tokens[index] = content["meta_info"]["completion_tokens"]
696
697
698

                    if not stream_buffer:  # The first chunk
                        if request.echo:
yichuan~'s avatar
yichuan~ committed
699
700
701
                            if isinstance(request.prompt, str):
                                # for the case of single str prompts
                                prompts = request.prompt
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
                            elif isinstance(request.prompt, list):
                                if isinstance(request.prompt[0], str):
                                    # for the case of multiple str prompts
                                    prompts = request.prompt[index // request.n]
                                elif isinstance(request.prompt[0], int):
                                    # for the case of single token ids prompt
                                    prompts = tokenizer_manager.tokenizer.decode(
                                        request.prompt, skip_special_tokens=True
                                    )
                                elif isinstance(request.prompt[0], list) and isinstance(
                                    request.prompt[0][0], int
                                ):
                                    # for the case of multiple token ids prompts
                                    prompts = tokenizer_manager.tokenizer.decode(
                                        request.prompt[index // request.n],
                                        skip_special_tokens=True,
                                    )
yichuan~'s avatar
yichuan~ committed
719

720
                            # Prepend prompt in response text.
yichuan~'s avatar
yichuan~ committed
721
                            text = prompts + text
722
723
724
725

                    if request.logprobs:
                        # The first chunk and echo is enabled.
                        if not stream_buffer and request.echo:
726
727
                            input_token_logprobs = content["meta_info"][
                                "input_token_logprobs"
728
                            ]
729
730
                            input_top_logprobs = content["meta_info"][
                                "input_top_logprobs"
731
732
                            ]
                        else:
733
734
                            input_token_logprobs = None
                            input_top_logprobs = None
735
736

                        logprobs = to_openai_style_logprobs(
737
738
739
740
                            input_token_logprobs=input_token_logprobs,
                            input_top_logprobs=input_top_logprobs,
                            output_token_logprobs=content["meta_info"][
                                "output_token_logprobs"
741
                            ][n_prev_token:],
742
743
                            output_top_logprobs=content["meta_info"][
                                "output_top_logprobs"
744
                            ][n_prev_token:],
745
                        )
746
                        n_prev_token = len(
747
                            content["meta_info"]["output_token_logprobs"]
748
                        )
749
                    else:
750
                        logprobs = None
751

752
                    delta = text[len(stream_buffer) :]
Liangsheng Yin's avatar
Liangsheng Yin committed
753
                    stream_buffer = stream_buffer + delta
754
                    choice_data = CompletionResponseStreamChoice(
755
                        index=index,
756
757
                        text=delta,
                        logprobs=logprobs,
758
759
760
                        finish_reason=format_finish_reason(
                            content["meta_info"]["finish_reason"]
                        ),
761
762
763
764
765
766
767
                    )
                    chunk = CompletionStreamResponse(
                        id=content["meta_info"]["id"],
                        object="text_completion",
                        choices=[choice_data],
                        model=request.model,
                    )
768
769
770
771

                    stream_buffers[index] = stream_buffer
                    n_prev_tokens[index] = n_prev_token

772
                    yield f"data: {chunk.model_dump_json()}\n\n"
773
                if request.stream_options and request.stream_options.include_usage:
774
775
776
777
778
779
780
781
                    total_prompt_tokens = sum(
                        tokens
                        for i, tokens in prompt_tokens.items()
                        if i % request.n == 0
                    )
                    total_completion_tokens = sum(
                        tokens for tokens in completion_tokens.values()
                    )
782
                    usage = UsageInfo(
783
784
785
                        prompt_tokens=total_prompt_tokens,
                        completion_tokens=total_completion_tokens,
                        total_tokens=total_prompt_tokens + total_completion_tokens,
786
787
788
789
790
791
792
793
794
795
796
797
                    )

                    final_usage_chunk = CompletionStreamResponse(
                        id=str(uuid.uuid4().hex),
                        choices=[],
                        model=request.model,
                        usage=usage,
                    )
                    final_usage_data = final_usage_chunk.model_dump_json(
                        exclude_unset=True, exclude_none=True
                    )
                    yield f"data: {final_usage_data}\n\n"
798
799
800
            except ValueError as e:
                error = create_streaming_error_response(str(e))
                yield f"data: {error}\n\n"
801
802
            yield "data: [DONE]\n\n"

803
804
805
806
807
        return StreamingResponse(
            generate_stream_resp(),
            media_type="text/event-stream",
            background=tokenizer_manager.create_abort_task(adapted_request),
        )
808
809

    # Non-streaming response.
810
811
    try:
        ret = await tokenizer_manager.generate_request(
812
813
            adapted_request, raw_request
        ).__anext__()
814
815
    except ValueError as e:
        return create_error_response(str(e))
816

817
818
819
    if not isinstance(ret, list):
        ret = [ret]

yichuan~'s avatar
yichuan~ committed
820
    response = v1_generate_response(request, ret, tokenizer_manager)
821
    return response
822

823

824
def v1_chat_generate_request(
825
826
827
    all_requests: List[ChatCompletionRequest],
    tokenizer_manager,
    request_ids: List[str] = None,
828
):
829
    input_ids = []
830
831
    sampling_params_list = []
    image_data_list = []
832
    return_logprobs = []
833
    logprob_start_lens = []
834
    top_logprobs_nums = []
835
836
837

    # NOTE: with openai API, the prompt's logprobs are always not computed

838
839
840
841
842
843
844
845
846
    for request in all_requests:
        # Prep the data needed for the underlying GenerateReqInput:
        #  - prompt: The full prompt string.
        #  - stop: Custom stop tokens.
        #  - image_data: None or a list of image strings (URLs or base64 strings).
        #    None skips any image processing in GenerateReqInput.
        if not isinstance(request.messages, str):
            # Apply chat template and its stop strings.
            if chat_template_name is None:
847
848
849
850
851
852
853
854
855
856
857
858
859
                openai_compatible_messages = []
                for message in request.messages:
                    if isinstance(message.content, str):
                        openai_compatible_messages.append(
                            {"role": message.role, "content": message.content}
                        )
                    else:
                        content_list = message.dict()["content"]
                        for content in content_list:
                            if content["type"] == "text":
                                openai_compatible_messages.append(
                                    {"role": message.role, "content": content["text"]}
                                )
860
                prompt_ids = tokenizer_manager.tokenizer.apply_chat_template(
861
862
863
                    openai_compatible_messages,
                    tokenize=True,
                    add_generation_prompt=True,
864
865
866
                )
                stop = request.stop
                image_data = None
867
            else:
868
869
870
871
872
873
874
875
876
                conv = generate_chat_conv(request, chat_template_name)
                prompt = conv.get_prompt()
                image_data = conv.image_data
                stop = conv.stop_str or []
                if request.stop:
                    if isinstance(request.stop, str):
                        stop.append(request.stop)
                    else:
                        stop.extend(request.stop)
877
                prompt_ids = tokenizer_manager.tokenizer.encode(prompt)
878
        else:
879
            # Use the raw prompt and stop strings if the messages is already a string.
yichuan~'s avatar
yichuan~ committed
880
            prompt_ids = request.messages
881
882
            stop = request.stop
            image_data = None
883
        input_ids.append(prompt_ids)
884
        return_logprobs.append(request.logprobs)
885
        logprob_start_lens.append(-1)
886
        top_logprobs_nums.append(request.top_logprobs)
887
888
889
890
        sampling_params_list.append(
            {
                "temperature": request.temperature,
                "max_new_tokens": request.max_tokens,
891
                "min_new_tokens": request.min_tokens,
892
                "stop": stop,
893
                "stop_token_ids": request.stop_token_ids,
894
895
896
                "top_p": request.top_p,
                "presence_penalty": request.presence_penalty,
                "frequency_penalty": request.frequency_penalty,
897
                "repetition_penalty": request.repetition_penalty,
898
                "regex": request.regex,
899
                "json_schema": request.json_schema,
900
901
                "n": request.n,
            }
902
        )
903
904
        image_data_list.append(image_data)
    if len(all_requests) == 1:
905
        input_ids = input_ids[0]
yichuan~'s avatar
yichuan~ committed
906
907
908
909
        if isinstance(input_ids, str):
            prompt_kwargs = {"text": input_ids}
        else:
            prompt_kwargs = {"input_ids": input_ids}
910
911
        sampling_params_list = sampling_params_list[0]
        image_data = image_data_list[0]
912
        return_logprobs = return_logprobs[0]
913
        logprob_start_lens = logprob_start_lens[0]
914
        top_logprobs_nums = top_logprobs_nums[0]
yichuan~'s avatar
yichuan~ committed
915
916
917
918
919
    else:
        if isinstance(input_ids[0], str):
            prompt_kwargs = {"text": input_ids}
        else:
            prompt_kwargs = {"input_ids": input_ids}
920

921
    adapted_request = GenerateReqInput(
yichuan~'s avatar
yichuan~ committed
922
        **prompt_kwargs,
923
924
        image_data=image_data,
        sampling_params=sampling_params_list,
925
        return_logprob=return_logprobs,
926
        logprob_start_len=logprob_start_lens,
927
928
929
        top_logprobs_num=top_logprobs_nums,
        stream=all_requests[0].stream,
        return_text_in_logprobs=True,
930
        rid=request_ids,
931
    )
932
933
934
    if len(all_requests) == 1:
        return adapted_request, all_requests[0]
    return adapted_request, all_requests
935

936

937
938
939
940
def v1_chat_generate_response(request, ret, to_file=False):
    choices = []

    for idx, ret_item in enumerate(ret):
941
        logprobs = False
yichuan~'s avatar
yichuan~ committed
942
        if isinstance(request, list) and request[idx].logprobs:
943
            logprobs = True
yichuan~'s avatar
yichuan~ committed
944
        elif (not isinstance(request, list)) and request.logprobs:
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
            logprobs = True
        if logprobs:
            logprobs = to_openai_style_logprobs(
                output_token_logprobs=ret_item["meta_info"]["output_token_logprobs"],
                output_top_logprobs=ret_item["meta_info"]["output_top_logprobs"],
            )
            token_logprobs = []
            for token, logprob in zip(logprobs.tokens, logprobs.token_logprobs):
                token_bytes = list(token.encode("utf-8"))
                top_logprobs = []
                if logprobs.top_logprobs:
                    for top_token, top_logprob in logprobs.top_logprobs[0].items():
                        top_token_bytes = list(top_token.encode("utf-8"))
                        top_logprobs.append(
                            TopLogprob(
                                token=top_token,
                                bytes=top_token_bytes,
                                logprob=top_logprob,
                            )
                        )
                token_logprobs.append(
                    ChatCompletionTokenLogprob(
                        token=token,
                        bytes=token_bytes,
                        logprob=logprob,
                        top_logprobs=top_logprobs,
                    )
                )

            choice_logprobs = ChoiceLogprobs(content=token_logprobs)
        else:
            choice_logprobs = None
977

978
        if to_file:
979
            # to make the choice data json serializable
980
981
982
            choice_data = {
                "index": 0,
                "message": {"role": "assistant", "content": ret_item["text"]},
983
                "logprobs": choice_logprobs,
984
985
986
                "finish_reason": format_finish_reason(
                    ret_item["meta_info"]["finish_reason"]
                ),
987
            }
988
        else:
989
990
991
            choice_data = ChatCompletionResponseChoice(
                index=idx,
                message=ChatMessage(role="assistant", content=ret_item["text"]),
992
                logprobs=choice_logprobs,
993
994
995
                finish_reason=format_finish_reason(
                    ret_item["meta_info"]["finish_reason"]
                ),
996
997
998
            )

        choices.append(choice_data)
999

1000
1001
1002
1003
1004
1005
1006
1007
    if to_file:
        responses = []

        for i, choice in enumerate(choices):
            response = {
                "status_code": 200,
                "request_id": ret[i]["meta_info"]["id"],
                "body": {
1008
                    # remain the same but if needed we can change that
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
                    "id": ret[i]["meta_info"]["id"],
                    "object": "chat.completion",
                    "created": int(time.time()),
                    "model": request[i].model,
                    "choices": choice,
                    "usage": {
                        "prompt_tokens": ret[i]["meta_info"]["prompt_tokens"],
                        "completion_tokens": ret[i]["meta_info"]["completion_tokens"],
                        "total_tokens": ret[i]["meta_info"]["prompt_tokens"]
                        + ret[i]["meta_info"]["completion_tokens"],
                    },
                    "system_fingerprint": None,
                },
            }
            responses.append(response)
        return responses
1025
    else:
1026
1027
1028
1029
        prompt_tokens = sum(
            ret[i]["meta_info"]["prompt_tokens"] for i in range(0, len(ret), request.n)
        )
        completion_tokens = sum(item["meta_info"]["completion_tokens"] for item in ret)
1030
1031
1032
1033
1034
        response = ChatCompletionResponse(
            id=ret[0]["meta_info"]["id"],
            model=request.model,
            choices=choices,
            usage=UsageInfo(
1035
1036
1037
                prompt_tokens=prompt_tokens,
                completion_tokens=completion_tokens,
                total_tokens=prompt_tokens + completion_tokens,
1038
1039
1040
            ),
        )
        return response
1041

1042
1043
1044
1045
1046

async def v1_chat_completions(tokenizer_manager, raw_request: Request):
    request_json = await raw_request.json()
    all_requests = [ChatCompletionRequest(**request_json)]
    adapted_request, request = v1_chat_generate_request(all_requests, tokenizer_manager)
1047
1048
1049
1050

    if adapted_request.stream:

        async def generate_stream_resp():
1051
1052
1053
1054
1055
            is_firsts = {}
            stream_buffers = {}
            n_prev_tokens = {}
            prompt_tokens = {}
            completion_tokens = {}
1056
            try:
1057
1058
1059
                async for content in tokenizer_manager.generate_request(
                    adapted_request, raw_request
                ):
1060
1061
1062
1063
1064
1065
1066
1067
                    index = content["index"]

                    is_first = is_firsts.get(index, True)
                    stream_buffer = stream_buffers.get(index, "")
                    n_prev_token = n_prev_tokens.get(index, 0)

                    prompt_tokens[index] = content["meta_info"]["prompt_tokens"]
                    completion_tokens[index] = content["meta_info"]["completion_tokens"]
yichuan~'s avatar
yichuan~ committed
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
                    if request.logprobs:
                        logprobs = to_openai_style_logprobs(
                            output_token_logprobs=content["meta_info"][
                                "output_token_logprobs"
                            ][n_prev_token:],
                            output_top_logprobs=content["meta_info"][
                                "output_top_logprobs"
                            ][n_prev_token:],
                        )

                        n_prev_token = len(
                            content["meta_info"]["output_token_logprobs"]
                        )
                        token_logprobs = []
                        for token, logprob in zip(
                            logprobs.tokens, logprobs.token_logprobs
                        ):
                            token_bytes = list(token.encode("utf-8"))
                            top_logprobs = []
                            if logprobs.top_logprobs:
                                for top_token, top_logprob in logprobs.top_logprobs[
                                    0
                                ].items():
                                    top_token_bytes = list(top_token.encode("utf-8"))
                                    top_logprobs.append(
                                        TopLogprob(
                                            token=top_token,
                                            bytes=top_token_bytes,
                                            logprob=top_logprob,
                                        )
                                    )
                            token_logprobs.append(
                                ChatCompletionTokenLogprob(
                                    token=token,
                                    bytes=token_bytes,
                                    logprob=logprob,
                                    top_logprobs=top_logprobs,
                                )
                            )

                        choice_logprobs = ChoiceLogprobs(content=token_logprobs)

                    else:
                        choice_logprobs = None

1113
1114
1115
1116
                    if is_first:
                        # First chunk with role
                        is_first = False
                        choice_data = ChatCompletionResponseStreamChoice(
1117
                            index=index,
1118
                            delta=DeltaMessage(role="assistant"),
1119
1120
1121
                            finish_reason=format_finish_reason(
                                content["meta_info"]["finish_reason"]
                            ),
yichuan~'s avatar
yichuan~ committed
1122
                            logprobs=choice_logprobs,
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
                        )
                        chunk = ChatCompletionStreamResponse(
                            id=content["meta_info"]["id"],
                            choices=[choice_data],
                            model=request.model,
                        )
                        yield f"data: {chunk.model_dump_json()}\n\n"

                    text = content["text"]
                    delta = text[len(stream_buffer) :]
Liangsheng Yin's avatar
Liangsheng Yin committed
1133
                    stream_buffer = stream_buffer + delta
1134
                    choice_data = ChatCompletionResponseStreamChoice(
1135
                        index=index,
1136
                        delta=DeltaMessage(content=delta),
1137
1138
1139
                        finish_reason=format_finish_reason(
                            content["meta_info"]["finish_reason"]
                        ),
yichuan~'s avatar
yichuan~ committed
1140
                        logprobs=choice_logprobs,
1141
1142
1143
1144
1145
1146
                    )
                    chunk = ChatCompletionStreamResponse(
                        id=content["meta_info"]["id"],
                        choices=[choice_data],
                        model=request.model,
                    )
1147
1148
1149
1150
1151

                    is_firsts[index] = is_first
                    stream_buffers[index] = stream_buffer
                    n_prev_tokens[index] = n_prev_token

1152
                    yield f"data: {chunk.model_dump_json()}\n\n"
1153
                if request.stream_options and request.stream_options.include_usage:
1154
1155
1156
1157
1158
1159
1160
1161
                    total_prompt_tokens = sum(
                        tokens
                        for i, tokens in prompt_tokens.items()
                        if i % request.n == 0
                    )
                    total_completion_tokens = sum(
                        tokens for tokens in completion_tokens.values()
                    )
1162
                    usage = UsageInfo(
1163
1164
1165
                        prompt_tokens=total_prompt_tokens,
                        completion_tokens=total_completion_tokens,
                        total_tokens=total_prompt_tokens + total_completion_tokens,
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
                    )

                    final_usage_chunk = ChatCompletionStreamResponse(
                        id=str(uuid.uuid4().hex),
                        choices=[],
                        model=request.model,
                        usage=usage,
                    )
                    final_usage_data = final_usage_chunk.model_dump_json(
                        exclude_unset=True, exclude_none=True
                    )
                    yield f"data: {final_usage_data}\n\n"
1178
1179
1180
            except ValueError as e:
                error = create_streaming_error_response(str(e))
                yield f"data: {error}\n\n"
1181
1182
            yield "data: [DONE]\n\n"

1183
1184
1185
1186
1187
        return StreamingResponse(
            generate_stream_resp(),
            media_type="text/event-stream",
            background=tokenizer_manager.create_abort_task(adapted_request),
        )
1188
1189

    # Non-streaming response.
1190
1191
    try:
        ret = await tokenizer_manager.generate_request(
1192
1193
            adapted_request, raw_request
        ).__anext__()
1194
1195
    except ValueError as e:
        return create_error_response(str(e))
1196
1197
1198
    if not isinstance(ret, list):
        ret = [ret]

1199
    response = v1_chat_generate_response(request, ret)
1200

1201
1202
1203
    return response


1204
1205
1206
def v1_embedding_request(all_requests, tokenizer_manager):
    prompts = []
    sampling_params_list = []
Ying Sheng's avatar
Ying Sheng committed
1207
    first_prompt_type = type(all_requests[0].input)
1208
1209

    for request in all_requests:
Ying Sheng's avatar
Ying Sheng committed
1210
        prompt = request.input
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
        assert (
            type(prompt) == first_prompt_type
        ), "All prompts must be of the same type in file input settings"
        prompts.append(prompt)

    if len(all_requests) == 1:
        prompt = prompts[0]
        if isinstance(prompt, str) or isinstance(prompt[0], str):
            prompt_kwargs = {"text": prompt}
        else:
            prompt_kwargs = {"input_ids": prompt}
    else:
        if isinstance(prompts[0], str) or isinstance(propmt[0][0], str):
            prompt_kwargs = {"text": prompts}
        else:
            prompt_kwargs = {"input_ids": prompts}

    adapted_request = EmbeddingReqInput(
        **prompt_kwargs,
    )

    if len(all_requests) == 1:
        return adapted_request, all_requests[0]
    return adapted_request, all_requests


Ying Sheng's avatar
Ying Sheng committed
1237
1238
1239
def v1_embedding_response(ret, model_path, to_file=False):
    embedding_objects = []
    prompt_tokens = 0
1240
    for idx, ret_item in enumerate(ret):
Ying Sheng's avatar
Ying Sheng committed
1241
1242
1243
        embedding_objects.append(
            EmbeddingObject(
                embedding=ret[idx]["embedding"],
1244
1245
1246
                index=idx,
            )
        )
Ying Sheng's avatar
Ying Sheng committed
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
        prompt_tokens += ret[idx]["meta_info"]["prompt_tokens"]

    return EmbeddingResponse(
        data=embedding_objects,
        model=model_path,
        usage=UsageInfo(
            prompt_tokens=prompt_tokens,
            total_tokens=prompt_tokens,
        ),
    )
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273


async def v1_embeddings(tokenizer_manager, raw_request: Request):
    request_json = await raw_request.json()
    all_requests = [EmbeddingRequest(**request_json)]
    adapted_request, request = v1_embedding_request(all_requests, tokenizer_manager)

    try:
        ret = await tokenizer_manager.generate_request(
            adapted_request, raw_request
        ).__anext__()
    except ValueError as e:
        return create_error_response(str(e))

    if not isinstance(ret, list):
        ret = [ret]

Ying Sheng's avatar
Ying Sheng committed
1274
    response = v1_embedding_response(ret, tokenizer_manager.model_path)
1275
1276
1277
1278

    return response


1279
def to_openai_style_logprobs(
1280
1281
1282
1283
    input_token_logprobs=None,
    output_token_logprobs=None,
    input_top_logprobs=None,
    output_top_logprobs=None,
1284
1285
1286
1287
1288
1289
1290
1291
):
    ret_logprobs = LogProbs()

    def append_token_logprobs(token_logprobs):
        for logprob, _, token_text in token_logprobs:
            ret_logprobs.tokens.append(token_text)
            ret_logprobs.token_logprobs.append(logprob)

1292
            # Not supported yet
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
            ret_logprobs.text_offset.append(-1)

    def append_top_logprobs(top_logprobs):
        for tokens in top_logprobs:
            if tokens is not None:
                ret_logprobs.top_logprobs.append(
                    {token[2]: token[0] for token in tokens}
                )
            else:
                ret_logprobs.top_logprobs.append(None)

1304
1305
1306
1307
1308
1309
1310
1311
    if input_token_logprobs is not None:
        append_token_logprobs(input_token_logprobs)
    if output_token_logprobs is not None:
        append_token_logprobs(output_token_logprobs)
    if input_top_logprobs is not None:
        append_top_logprobs(input_top_logprobs)
    if output_top_logprobs is not None:
        append_top_logprobs(output_top_logprobs)
1312

Liangsheng Yin's avatar
Liangsheng Yin committed
1313
    return ret_logprobs