benchmark_serving.py 42.3 KB
Newer Older
1
r"""Benchmark online serving throughput.
2
3

On the server side, run one of the following commands:
4
    vLLM OpenAI API server
Ethan Xu's avatar
Ethan Xu committed
5
6
    vllm serve <your_model> \
        --swap-space 16 \
7
        --disable-log-requests
8
9

    (TGI backend)
Ronen Schaffer's avatar
Ronen Schaffer committed
10
    ./launch_tgi_server.sh <your_model> <max_batch_total_tokens>
11
12
13
14

On the client side, run:
    python benchmarks/benchmark_serving.py \
        --backend <backend> \
15
16
17
18
19
        --model <your_model> \
        --dataset-name sharegpt \
        --dataset-path <path to dataset> \
        --request-rate <request_rate> \ # By default <request_rate> is inf
        --num-prompts <num_prompts> # By default <num_prompts> is 1000
20

21
22
23
    when using tgi backend, add
        --endpoint /generate_stream
    to the end of the command above.
24
25
26
"""
import argparse
import asyncio
27
28
import base64
import io
29
import json
30
import os
31
32
import random
import time
33
import warnings
34
35
from dataclasses import dataclass
from datetime import datetime
36
from typing import Any, AsyncGenerator, Collection, Dict, List, Optional, Tuple
37
38

import numpy as np
39
40
from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
                                  RequestFuncOutput)
41
42
from datasets import load_dataset
from PIL.Image import Image
43
from tqdm.asyncio import tqdm
44
from transformers import PreTrainedTokenizerBase
45

46
47
48
49
try:
    from vllm.transformers_utils.tokenizer import get_tokenizer
except ImportError:
    from backend_request_func import get_tokenizer
50

51
52
53
54
55
try:
    from vllm.utils import FlexibleArgumentParser
except ImportError:
    from argparse import ArgumentParser as FlexibleArgumentParser

56
57
MILLISECONDS_TO_SECONDS_CONVERSION = 1000

58
59
60
61
62
63
64

@dataclass
class BenchmarkMetrics:
    completed: int
    total_input: int
    total_output: int
    request_throughput: float
65
    request_goodput: float
66
    output_throughput: float
67
    total_token_throughput: float
68
69
    mean_ttft_ms: float
    median_ttft_ms: float
70
    std_ttft_ms: float
71
    percentiles_ttft_ms: List[Tuple[float, float]]
72
73
    mean_tpot_ms: float
    median_tpot_ms: float
74
    std_tpot_ms: float
75
    percentiles_tpot_ms: List[Tuple[float, float]]
76
77
    mean_itl_ms: float
    median_itl_ms: float
78
    std_itl_ms: float
79
80
81
82
83
84
85
86
    percentiles_itl_ms: List[Tuple[float, float]]
    # E2EL stands for end-to-end latency per request.
    # It is the time taken on the client side from sending
    # a request to receiving a complete response.
    mean_e2el_ms: float
    median_e2el_ms: float
    std_e2el_ms: float
    percentiles_e2el_ms: List[Tuple[float, float]]
87
88


89
def sample_sharegpt_requests(
90
91
92
    dataset_path: str,
    num_requests: int,
    tokenizer: PreTrainedTokenizerBase,
93
    fixed_output_len: Optional[int] = None,
94
) -> List[Tuple[str, int, int, None]]:
95
    # Load the dataset.
96
    with open(dataset_path, encoding='utf-8') as f:
97
98
        dataset = json.load(f)
    # Filter out the conversations with less than 2 turns.
99
    dataset = [data for data in dataset if len(data["conversations"]) >= 2]
100
    # Only keep the first two turns of each conversation.
101
102
    dataset = [(data["conversations"][0]["value"],
                data["conversations"][1]["value"]) for data in dataset]
103

104
105
    # Shuffle the dataset.
    random.shuffle(dataset)
106

107
    # Filter out sequences that are too long or too short
108
    filtered_dataset: List[Tuple[str, int, int]] = []
109
110
111
112
113
114
115
116
117
    for i in range(len(dataset)):
        if len(filtered_dataset) == num_requests:
            break

        # Tokenize the prompts and completions.
        prompt = dataset[i][0]
        prompt_token_ids = tokenizer(prompt).input_ids
        completion = dataset[i][1]
        completion_token_ids = tokenizer(completion).input_ids
118
        prompt_len = len(prompt_token_ids)
119
120
        output_len = len(completion_token_ids
                         ) if fixed_output_len is None else fixed_output_len
121
        if prompt_len < 4 or (fixed_output_len is None and output_len < 4):
122
123
124
125
126
            # Prune too short sequences.
            continue
        if prompt_len > 1024 or prompt_len + output_len > 2048:
            # Prune too long sequences.
            continue
127
        filtered_dataset.append((prompt, prompt_len, output_len, None))
128

129
    return filtered_dataset
130
131


132
133
134
135
136
137
138
def sample_sonnet_requests(
    dataset_path: str,
    num_requests: int,
    input_len: int,
    output_len: int,
    prefix_len: int,
    tokenizer: PreTrainedTokenizerBase,
139
) -> List[Tuple[str, str, int, int, None]]:
140
141
142
    assert (
        input_len > prefix_len
    ), "'args.sonnet-input-len' must be greater than 'args.prefix-input-len'."
143
144

    # Load the dataset.
145
    with open(dataset_path, encoding='utf-8') as f:
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
        poem_lines = f.readlines()

    # Tokenize the poem lines.
    poem_token_ids = tokenizer(poem_lines).input_ids
    average_poem_len = sum(
        len(token_ids) for token_ids in poem_token_ids) / len(poem_token_ids)

    # Base prefix for all requests.
    base_prompt = "Pick as many lines as you can from these poem lines:\n"
    base_message = [{
        "role": "user",
        "content": base_prompt,
    }]
    base_prompt_formatted = tokenizer.apply_chat_template(
        base_message, add_generation_prompt=True, tokenize=False)
    base_prompt_offset = len(tokenizer(base_prompt_formatted).input_ids)

163
164
165
    assert (
        input_len > base_prompt_offset
    ), f"Please set 'args.sonnet-input-len' higher than {base_prompt_offset}."
166
167
168
169
170
171
172
    num_input_lines = round(
        (input_len - base_prompt_offset) / average_poem_len)

    # First approximately `prefix_len` number of tokens in the
    # prompt are fixed poem lines.
    assert (
        prefix_len > base_prompt_offset
173
    ), f"Please set 'args.sonnet-prefix-len' higher than {base_prompt_offset}."
174
175
176
177
178
179
180
181

    num_prefix_lines = round(
        (prefix_len - base_prompt_offset) / average_poem_len)
    prefix_lines = poem_lines[:num_prefix_lines]

    # Sample the rest of lines per request.
    sampled_requests: List[Tuple[str, int, int]] = []
    for _ in range(num_requests):
182
183
184
        num_lines_needed = num_input_lines - num_prefix_lines
        sampled_lines = "".join(prefix_lines +
                                random.choices(poem_lines, k=num_lines_needed))
185
186
187
188
189
190
191
192
193
194
195
196

        prompt = f"{base_prompt}{sampled_lines}"
        message = [
            {
                "role": "user",
                "content": prompt,
            },
        ]
        prompt_formatted = tokenizer.apply_chat_template(
            message, add_generation_prompt=True, tokenize=False)
        prompt_len = len(tokenizer(prompt_formatted).input_ids)
        sampled_requests.append(
197
198
199
200
201
202
203
204
205
206
207
            (prompt, prompt_formatted, prompt_len, output_len, None))

    return sampled_requests


def sample_hf_requests(
    dataset_path: str,
    dataset_subset: str,
    dataset_split: str,
    num_requests: int,
    tokenizer: PreTrainedTokenizerBase,
208
    random_seed: int,
209
210
211
212
213
214
215
216
    fixed_output_len: Optional[int] = None,
) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:
    dataset = load_dataset(dataset_path,
                           name=dataset_subset,
                           split=dataset_split,
                           streaming=True)
    assert "conversations" in dataset.features, (
        "HF Dataset must have 'conversations' column.")
217
218
    filter_func = lambda x: len(x["conversations"]) >= 2
    filtered_dataset = dataset.shuffle(seed=random_seed).filter(filter_func)
219
220
221
222
223
224
225
226
227
228
229
230
231
232
    sampled_requests: List[Tuple[str, int, int, Dict[str,
                                                     Collection[str]]]] = []
    for data in filtered_dataset:
        if len(sampled_requests) == num_requests:
            break

        # Tokenize the prompts and completions.
        prompt = data["conversations"][0]["value"]
        prompt_token_ids = tokenizer(prompt).input_ids
        completion = data["conversations"][1]["value"]
        completion_token_ids = tokenizer(completion).input_ids
        prompt_len = len(prompt_token_ids)
        output_len = len(completion_token_ids
                         ) if fixed_output_len is None else fixed_output_len
233
        if fixed_output_len is None and (prompt_len < 4 or output_len < 4):
234
235
            # Prune too short sequences.
            continue
236
237
        if fixed_output_len is None and \
            (prompt_len > 1024 or prompt_len + output_len > 2048):
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
            # Prune too long sequences.
            continue

        if "image" in data and isinstance(data["image"], Image):
            image: Image = data["image"]
            image = image.convert("RGB")
            image_data = io.BytesIO()
            image.save(image_data, format='JPEG')
            image_base64 = base64.b64encode(
                image_data.getvalue()).decode("utf-8")
            mm_content = {
                "type": "image_url",
                "image_url": {
                    "url": f"data:image/jpeg;base64,{image_base64}"
                },
            }
        else:
            mm_content = None

        sampled_requests.append((prompt, prompt_len, output_len, mm_content))
258
259
260
261

    return sampled_requests


262
def sample_random_requests(
263
264
265
266
267
268
269
270
271
272
    prefix_len: int,
    input_len: int,
    output_len: int,
    num_prompts: int,
    range_ratio: float,
    tokenizer: PreTrainedTokenizerBase,
) -> List[Tuple[str, int, int]]:
    prefix_token_ids = np.random.randint(0,
                                         tokenizer.vocab_size,
                                         size=prefix_len).tolist()
273
274
275
276
277
278
279
280
281
282
283
284
285

    input_lens = np.random.randint(
        int(input_len * range_ratio),
        input_len + 1,
        size=num_prompts,
    )
    output_lens = np.random.randint(
        int(output_len * range_ratio),
        output_len + 1,
        size=num_prompts,
    )
    offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
    input_requests = []
286
    for i in range(num_prompts):
287
288
        prompt = tokenizer.decode(prefix_token_ids +
                                  [(offsets[i] + i + j) % tokenizer.vocab_size
289
                                   for j in range(input_lens[i])])
290

291
292
        input_requests.append((prompt, int(prefix_len + input_lens[i]),
                               int(output_lens[i]), None))
293
294
295
296

    return input_requests


297
298
299
async def get_request(
    input_requests: List[Tuple[str, int, int]],
    request_rate: float,
300
    burstiness: float = 1.0,
301
) -> AsyncGenerator[Tuple[str, int, int], None]:
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
    """
    Asynchronously generates requests at a specified rate 
    with OPTIONAL burstiness.
    
    Args:
        input_requests: 
            A list of input requests, each represented as a tuple.
        request_rate: 
            The rate at which requests are generated (requests/s).
        burstiness (optional): 
            The burstiness factor of the request generation. 
            Only takes effect when request_rate is not inf.
            Default value is 1, which follows a Poisson process.
            Otherwise, the request intervals follow a gamma distribution.
            A lower burstiness value (0 < burstiness < 1) results 
            in more bursty requests, while a higher burstiness value 
            (burstiness > 1) results in a more uniform arrival of requests.
    """
320
    input_requests = iter(input_requests)
321
322
323
324
325
326

    # Calculate scale parameter theta to maintain the desired request_rate.
    assert burstiness > 0, (
        f"A positive burstiness factor is expected, but given {burstiness}.")
    theta = 1.0 / (request_rate * burstiness)

327
328
329
330
331
332
    for request in input_requests:
        yield request

        if request_rate == float("inf"):
            # If the request rate is infinity, then we don't need to wait.
            continue
333

334
335
336
        # Sample the request interval from the gamma distribution.
        # If burstiness is 1, it follows exponential distribution.
        interval = np.random.gamma(shape=burstiness, scale=theta)
337
338
339
340
        # The next request will be sent after the interval.
        await asyncio.sleep(interval)


341
342
343
344
345
def calculate_metrics(
    input_requests: List[Tuple[str, int, int]],
    outputs: List[RequestFuncOutput],
    dur_s: float,
    tokenizer: PreTrainedTokenizerBase,
346
347
    selected_percentile_metrics: List[str],
    selected_percentiles: List[float],
348
    gootput_config_dict: Dict[str, float],
349
) -> Tuple[BenchmarkMetrics, List[int]]:
350
    actual_output_lens: List[int] = []
351
352
    total_input = 0
    completed = 0
353
    good_completed = 0
354
355
    itls: List[float] = []
    tpots: List[float] = []
356
    all_tpots: List[float] = []
357
    ttfts: List[float] = []
358
    e2els: List[float] = []
359
360
    for i in range(len(outputs)):
        if outputs[i].success:
361
362
363
            # We use the tokenizer to count the number of output tokens for all
            # serving backends instead of looking at len(outputs[i].itl) since
            # multiple output tokens may be bundled together
364
            # Note : this may inflate the output token count slightly
365
366
367
            output_len = len(
                tokenizer(outputs[i].generated_text,
                          add_special_tokens=False).input_ids)
368
            actual_output_lens.append(output_len)
369
            total_input += input_requests[i][1]
370
            tpot = 0
371
            if output_len > 1:
372
373
374
375
376
                tpot = (outputs[i].latency - outputs[i].ttft) / (output_len -
                                                                 1)
                tpots.append(tpot)
            # Note: if output_len <= 1, we regard tpot as 0 for goodput
            all_tpots.append(tpot)
377
            itls += outputs[i].itl
378
            ttfts.append(outputs[i].ttft)
379
            e2els.append(outputs[i].latency)
380
            completed += 1
381
382
        else:
            actual_output_lens.append(0)
383

384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
    if gootput_config_dict:
        valid_metrics = []
        slo_values = []

        if "ttft" in gootput_config_dict:
            valid_metrics.append(ttfts)
            slo_values.append(gootput_config_dict["ttft"] /
                              MILLISECONDS_TO_SECONDS_CONVERSION)
        if "tpot" in gootput_config_dict:
            valid_metrics.append(all_tpots)
            slo_values.append(gootput_config_dict["tpot"] /
                              MILLISECONDS_TO_SECONDS_CONVERSION)
        if "e2el" in gootput_config_dict:
            valid_metrics.append(e2els)
            slo_values.append(gootput_config_dict["e2el"] /
                              MILLISECONDS_TO_SECONDS_CONVERSION)

        for req_metric in zip(*valid_metrics):
            is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)])
            if is_good_req:
                good_completed += 1

406
407
408
409
410
    if completed == 0:
        warnings.warn(
            "All requests failed. This is likely due to a misconfiguration "
            "on the benchmark arguments.",
            stacklevel=2)
411
412
413
    metrics = BenchmarkMetrics(
        completed=completed,
        total_input=total_input,
414
        total_output=sum(actual_output_lens),
415
        request_throughput=completed / dur_s,
416
        request_goodput=good_completed / dur_s,
417
        output_throughput=sum(actual_output_lens) / dur_s,
418
        total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
419
420
        mean_ttft_ms=np.mean(ttfts or 0) *
        1000,  # ttfts is empty if streaming is not supported by backend
421
        std_ttft_ms=np.std(ttfts or 0) * 1000,
422
423
424
        median_ttft_ms=np.median(ttfts or 0) * 1000,
        percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000)
                             for p in selected_percentiles],
425
        mean_tpot_ms=np.mean(tpots or 0) * 1000,
426
        std_tpot_ms=np.std(tpots or 0) * 1000,
427
428
429
        median_tpot_ms=np.median(tpots or 0) * 1000,
        percentiles_tpot_ms=[(p, np.percentile(tpots or 0, p) * 1000)
                             for p in selected_percentiles],
430
        mean_itl_ms=np.mean(itls or 0) * 1000,
431
        std_itl_ms=np.std(itls or 0) * 1000,
432
433
434
        median_itl_ms=np.median(itls or 0) * 1000,
        percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000)
                            for p in selected_percentiles],
435
        mean_e2el_ms=np.mean(e2els or 0) * 1000,
436
        std_e2el_ms=np.std(e2els or 0) * 1000,
437
        median_e2el_ms=np.median(e2els or 0) * 1000,
438
439
        percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)
                             for p in selected_percentiles],
440
    )
441

442
    return metrics, actual_output_lens
443

444
445
446
447

async def benchmark(
    backend: str,
    api_url: str,
448
    base_url: str,
449
450
    model_id: str,
    tokenizer: PreTrainedTokenizerBase,
451
    input_requests: List[Tuple[str, int, int]],
452
    logprobs: Optional[int],
453
454
    best_of: int,
    request_rate: float,
455
    burstiness: float,
456
    disable_tqdm: bool,
457
    profile: bool,
458
459
    selected_percentile_metrics: List[str],
    selected_percentiles: List[str],
460
    ignore_eos: bool,
461
    gootput_config_dict: Dict[str, float],
462
    max_concurrency: Optional[int],
463
464
):
    if backend in ASYNC_REQUEST_FUNCS:
465
        request_func = ASYNC_REQUEST_FUNCS[backend]
466
467
468
    else:
        raise ValueError(f"Unknown backend: {backend}")

469
    print("Starting initial single prompt test run...")
470
471
472
473
474
475
    test_prompt, test_prompt_len, test_output_len, test_mm_content = (
        input_requests[0])
    if backend != "openai-chat" and test_mm_content is not None:
        # multi-modal benchmark is only available on OpenAI Chat backend.
        raise ValueError(
            "Multi-modal content is only supported on 'openai-chat' backend.")
476
477
478
479
480
481
    test_input = RequestFuncInput(
        model=model_id,
        prompt=test_prompt,
        api_url=api_url,
        prompt_len=test_prompt_len,
        output_len=test_output_len,
482
        logprobs=logprobs,
483
        best_of=best_of,
484
        multi_modal_content=test_mm_content,
485
        ignore_eos=ignore_eos,
486
487
488
489
490
491
492
493
    )
    test_output = await request_func(request_func_input=test_input)
    if not test_output.success:
        raise ValueError(
            "Initial test run failed - Please make sure benchmark arguments "
            f"are correctly specified. Error: {test_output.error}")
    else:
        print("Initial test run completed. Starting main benchmark run...")
494
495
496

    if profile:
        print("Starting profiler...")
497
498
499
500
501
502
503
504
505
        profile_input = RequestFuncInput(model=model_id,
                                         prompt=test_prompt,
                                         api_url=base_url + "/start_profile",
                                         prompt_len=test_prompt_len,
                                         output_len=test_output_len,
                                         logprobs=logprobs,
                                         best_of=best_of,
                                         multi_modal_content=test_mm_content,
                                         ignore_eos=ignore_eos)
506
507
508
509
        profile_output = await request_func(request_func_input=profile_input)
        if profile_output.success:
            print("Profiler started")

510
511
512
513
514
    if burstiness == 1.0:
        distribution = "Poisson process"
    else:
        distribution = "Gamma distribution"

515
    print(f"Traffic request rate: {request_rate}")
516
    print(f"Burstiness factor: {burstiness} ({distribution})")
517
    print(f"Maximum request concurrency: {max_concurrency}")
518

519
520
    pbar = None if disable_tqdm else tqdm(total=len(input_requests))

521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
    # This can be used once the minimum Python version is 3.10 or higher,
    # and it will simplify the code in limited_request_func.
    #    semaphore = (asyncio.Semaphore(max_concurrency)
    #                 if max_concurrency else contextlib.nullcontext())
    semaphore = (asyncio.Semaphore(max_concurrency)
                 if max_concurrency else None)

    async def limited_request_func(request_func_input, pbar):
        if semaphore is None:
            return await request_func(request_func_input=request_func_input,
                                      pbar=pbar)
        async with semaphore:
            return await request_func(request_func_input=request_func_input,
                                      pbar=pbar)

536
    benchmark_start_time = time.perf_counter()
537
    tasks: List[asyncio.Task] = []
538
    async for request in get_request(input_requests, request_rate, burstiness):
539
        prompt, prompt_len, output_len, mm_content = request
540
541
542
543
544
545
546
547
548
        request_func_input = RequestFuncInput(model=model_id,
                                              prompt=prompt,
                                              api_url=api_url,
                                              prompt_len=prompt_len,
                                              output_len=output_len,
                                              logprobs=logprobs,
                                              best_of=best_of,
                                              multi_modal_content=mm_content,
                                              ignore_eos=ignore_eos)
549
550
        tasks.append(
            asyncio.create_task(
551
552
                limited_request_func(request_func_input=request_func_input,
                                     pbar=pbar)))
553
    outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
554

555
556
557
558
559
560
561
562
    if profile:
        print("Stopping profiler...")
        profile_input = RequestFuncInput(
            model=model_id,
            prompt=test_prompt,
            api_url=base_url + "/stop_profile",
            prompt_len=test_prompt_len,
            output_len=test_output_len,
563
            logprobs=logprobs,
564
565
566
567
568
569
            best_of=best_of,
        )
        profile_output = await request_func(request_func_input=profile_input)
        if profile_output.success:
            print("Profiler stopped")

570
    if pbar is not None:
571
572
573
574
        pbar.close()

    benchmark_duration = time.perf_counter() - benchmark_start_time

575
    metrics, actual_output_lens = calculate_metrics(
576
577
578
579
        input_requests=input_requests,
        outputs=outputs,
        dur_s=benchmark_duration,
        tokenizer=tokenizer,
580
581
        selected_percentile_metrics=selected_percentile_metrics,
        selected_percentiles=selected_percentiles,
582
        gootput_config_dict=gootput_config_dict,
583
584
    )

585
586
587
588
589
590
591
592
593
    print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
    print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
    print("{:<40} {:<10.2f}".format("Benchmark duration (s):",
                                    benchmark_duration))
    print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
    print("{:<40} {:<10}".format("Total generated tokens:",
                                 metrics.total_output))
    print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
                                    metrics.request_throughput))
594
595
596
    if gootput_config_dict:
        print("{:<40} {:<10.2f}".format("Request goodput (req/s):",
                                        metrics.request_goodput))
597
598
    print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
                                    metrics.output_throughput))
599
600
    print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
                                    metrics.total_token_throughput))
601
602
603
604
605
606

    result = {
        "duration": benchmark_duration,
        "completed": metrics.completed,
        "total_input_tokens": metrics.total_input,
        "total_output_tokens": metrics.total_output,
607
        "request_throughput": metrics.request_throughput,
608
609
        "request_goodput:":
        metrics.request_goodput if gootput_config_dict else None,
610
        "output_throughput": metrics.output_throughput,
611
        "total_token_throughput": metrics.total_token_throughput,
612
613
614
615
616
617
        "input_lens": [output.prompt_len for output in outputs],
        "output_lens": actual_output_lens,
        "ttfts": [output.ttft for output in outputs],
        "itls": [output.itl for output in outputs],
        "generated_texts": [output.generated_text for output in outputs],
        "errors": [output.error for output in outputs],
618
    }
619
620
621
622
623
624
625
626
627

    def process_one_metric(
        # E.g., "ttft"
        metric_attribute_name: str,
        # E.g., "TTFT"
        metric_name: str,
        # E.g., "Time to First Token"
        metric_header: str,
    ):
628
        # This function prints and adds statistics of the specified
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
        # metric.
        if metric_attribute_name not in selected_percentile_metrics:
            return
        print("{s:{c}^{n}}".format(s=metric_header, n=50, c='-'))
        print("{:<40} {:<10.2f}".format(
            f"Mean {metric_name} (ms):",
            getattr(metrics, f"mean_{metric_attribute_name}_ms")))
        print("{:<40} {:<10.2f}".format(
            f"Median {metric_name} (ms):",
            getattr(metrics, f"median_{metric_attribute_name}_ms")))
        result[f"mean_{metric_attribute_name}_ms"] = getattr(
            metrics, f"mean_{metric_attribute_name}_ms")
        result[f"median_{metric_attribute_name}_ms"] = getattr(
            metrics, f"median_{metric_attribute_name}_ms")
        result[f"std_{metric_attribute_name}_ms"] = getattr(
            metrics, f"std_{metric_attribute_name}_ms")
        for p, value in getattr(metrics,
                                f"percentiles_{metric_attribute_name}_ms"):
            p_word = str(int(p)) if int(p) == p else str(p)
            print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):",
                                            value))
            result[f"p{p_word}_{metric_attribute_name}_ms"] = value

    process_one_metric("ttft", "TTFT", "Time to First Token")
    process_one_metric("tpot", "TPOT",
                       "Time per Output Token (excl. 1st token)")
    process_one_metric("itl", "ITL", "Inter-token Latency")
    process_one_metric("e2el", "E2EL", "End-to-end Latency")

    print("=" * 50)

660
    return result
661
662


663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
def check_goodput_args(args):
    # Check and parse goodput arguments
    gootput_config_dict = {}
    VALID_NAMES = ["ttft", "tpot", "e2el"]
    if args.goodput:
        gootput_config_dict = parse_goodput(args.goodput)
        for slo_name, slo_val in gootput_config_dict.items():
            if slo_name not in VALID_NAMES:
                raise ValueError(
                    f"Invalid metric name found, {slo_name}: {slo_val}. "
                    "The service level objective name should be one of "
                    f"{str(VALID_NAMES)}. ")
            if slo_val < 0:
                raise ValueError(
                    f"Invalid value found, {slo_name}: {slo_val}. "
                    "The service level objective value should be "
                    "non-negative.")
    return gootput_config_dict


def parse_goodput(slo_pairs):
    gootput_config_dict = {}
    try:
        for slo_pair in slo_pairs:
            slo_name, slo_val = slo_pair.split(":")
            gootput_config_dict[slo_name] = float(slo_val)
    except ValueError as err:
        raise argparse.ArgumentTypeError(
            "Invalid format found for service level objectives. "
            "Specify service level objectives for goodput as \"KEY:VALUE\" "
            "pairs, where the key is a metric name, and the value is a "
            "number in milliseconds.") from err
    return gootput_config_dict


698
699
700
701
702
def main(args: argparse.Namespace):
    print(args)
    random.seed(args.seed)
    np.random.seed(args.seed)

703
704
705
706
707
708
    backend = args.backend
    model_id = args.model
    tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model

    if args.base_url is not None:
        api_url = f"{args.base_url}{args.endpoint}"
709
        base_url = f"{args.base_url}"
710
711
    else:
        api_url = f"http://{args.host}:{args.port}{args.endpoint}"
712
        base_url = f"http://{args.host}:{args.port}"
713
714

    tokenizer = get_tokenizer(tokenizer_id,
715
                              trust_remote_code=args.trust_remote_code)
716
717
718
719
720
721
722
723
724
725
726

    if args.dataset is not None:
        warnings.warn(
            "The '--dataset' argument will be deprecated in the next "
            "release. Please use '--dataset-name' and "
            "'--dataset-path' in the future runs.",
            stacklevel=2)
        input_requests = sample_sharegpt_requests(
            dataset_path=args.dataset,
            num_requests=args.num_prompts,
            tokenizer=tokenizer,
727
            fixed_output_len=args.sharegpt_output_len,
728
729
730
731
732
733
734
        )

    elif args.dataset_name == "sharegpt":
        input_requests = sample_sharegpt_requests(
            dataset_path=args.dataset_path,
            num_requests=args.num_prompts,
            tokenizer=tokenizer,
735
            fixed_output_len=args.sharegpt_output_len,
736
737
738
739
740
741
742
743
        )

    elif args.dataset_name == "sonnet":
        # Do not format the prompt, pass to message directly
        if args.backend == "openai-chat":
            input_requests = sample_sonnet_requests(
                dataset_path=args.dataset_path,
                num_requests=args.num_prompts,
744
745
746
                input_len=args.sonnet_input_len,
                output_len=args.sonnet_output_len,
                prefix_len=args.sonnet_prefix_len,
747
748
                tokenizer=tokenizer,
            )
749
            input_requests = [(prompt, prompt_len, output_len, None)
750
                              for prompt, prompt_formatted, prompt_len,
751
                              output_len, _ in input_requests]
752
753
754
755
756
757
758
        else:
            assert (
                tokenizer.chat_template or tokenizer.default_chat_template
            ), "Tokenizer/model must have chat template for sonnet dataset."
            input_requests = sample_sonnet_requests(
                dataset_path=args.dataset_path,
                num_requests=args.num_prompts,
759
760
761
                input_len=args.sonnet_input_len,
                output_len=args.sonnet_output_len,
                prefix_len=args.sonnet_prefix_len,
762
763
                tokenizer=tokenizer,
            )
764
            input_requests = [(prompt_formatted, prompt_len, output_len, None)
765
                              for prompt, prompt_formatted, prompt_len,
766
                              output_len, _ in input_requests]
767

768
769
770
771
772
773
774
    elif args.dataset_name == "hf":
        input_requests = sample_hf_requests(
            dataset_path=args.dataset_path,
            dataset_subset=args.hf_subset,
            dataset_split=args.hf_split,
            num_requests=args.num_prompts,
            tokenizer=tokenizer,
775
            random_seed=args.seed,
776
777
778
            fixed_output_len=args.hf_output_len,
        )

779
780
    elif args.dataset_name == "random":
        input_requests = sample_random_requests(
781
            prefix_len=args.random_prefix_len,
782
783
            input_len=args.random_input_len,
            output_len=args.random_output_len,
784
            num_prompts=args.num_prompts,
785
            range_ratio=args.random_range_ratio,
786
787
788
            tokenizer=tokenizer,
        )

789
790
    else:
        raise ValueError(f"Unknown dataset: {args.dataset_name}")
791

792
793
    gootput_config_dict = check_goodput_args(args)

794
795
796
797
    benchmark_result = asyncio.run(
        benchmark(
            backend=backend,
            api_url=api_url,
798
            base_url=base_url,
799
800
801
            model_id=model_id,
            tokenizer=tokenizer,
            input_requests=input_requests,
802
            logprobs=args.logprobs,
803
804
            best_of=args.best_of,
            request_rate=args.request_rate,
805
            burstiness=args.burstiness,
806
            disable_tqdm=args.disable_tqdm,
807
            profile=args.profile,
808
809
810
811
            selected_percentile_metrics=args.percentile_metrics.split(","),
            selected_percentiles=[
                float(p) for p in args.metric_percentiles.split(",")
            ],
812
            ignore_eos=args.ignore_eos,
813
            gootput_config_dict=gootput_config_dict,
814
            max_concurrency=args.max_concurrency,
815
816
817
818
        ))

    # Save config and results to json
    if args.save_result:
819
        result_json: Dict[str, Any] = {}
820
821
822
823
824
825
826
827
828
829

        # Setup
        current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
        result_json["date"] = current_dt
        result_json["backend"] = backend
        result_json["model_id"] = model_id
        result_json["tokenizer_id"] = tokenizer_id
        result_json["best_of"] = args.best_of
        result_json["num_prompts"] = args.num_prompts

830
831
832
833
834
835
836
837
838
839
840
        # Metadata
        if args.metadata:
            for item in args.metadata:
                if "=" in item:
                    kvstring = item.split("=")
                    result_json[kvstring[0].strip()] = kvstring[1].strip()
                else:
                    raise ValueError(
                        "Invalid metadata format. Please use KEY=VALUE format."
                    )

841
842
843
        # Traffic
        result_json["request_rate"] = (
            args.request_rate if args.request_rate < float("inf") else "inf")
844
        result_json["burstiness"] = args.burstiness
845
        result_json["max_concurrency"] = args.max_concurrency
846
847
848
849
850
851

        # Merge with benchmark result
        result_json = {**result_json, **benchmark_result}

        # Save to file
        base_model_id = model_id.split("/")[-1]
852
853
854
        max_concurrency_str = (f"-concurrency{args.max_concurrency}"
                               if args.max_concurrency is not None else "")
        file_name = f"{backend}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json"  #noqa
855
856
        if args.result_filename:
            file_name = args.result_filename
857
858
        if args.result_dir:
            file_name = os.path.join(args.result_dir, file_name)
859
        with open(file_name, "w", encoding='utf-8') as outfile:
860
            json.dump(result_json, outfile)
861
862
863


if __name__ == "__main__":
864
    parser = FlexibleArgumentParser(
865
        description="Benchmark the online serving throughput.")
866
867
868
869
870
871
872
873
874
875
876
877
    parser.add_argument(
        "--backend",
        type=str,
        default="vllm",
        choices=list(ASYNC_REQUEST_FUNCS.keys()),
    )
    parser.add_argument(
        "--base-url",
        type=str,
        default=None,
        help="Server or API base url if not using http host and port.",
    )
878
    parser.add_argument("--host", type=str, default="localhost")
879
    parser.add_argument("--port", type=int, default=8000)
880
881
882
    parser.add_argument(
        "--endpoint",
        type=str,
883
        default="/v1/completions",
884
885
        help="API endpoint.",
    )
886
887
888
889
890
891
892
893
894
895
896
    parser.add_argument(
        "--dataset",
        type=str,
        default=None,
        help="Path to the ShareGPT dataset, will be deprecated in the "
        "next release.",
    )
    parser.add_argument(
        "--dataset-name",
        type=str,
        default="sharegpt",
897
        choices=["sharegpt", "sonnet", "random", "hf"],
898
899
900
        help="Name of the dataset to benchmark on.",
    )
    parser.add_argument("--dataset-path",
901
                        type=str,
902
                        default=None,
903
904
                        help="Path to the sharegpt/sonnet dataset. "
                        "Or the huggingface dataset ID if using HF dataset.")
905
906
907
908
909
910
911
912
913
914
915
916
917
    parser.add_argument(
        "--max-concurrency",
        type=int,
        default=None,
        help="Maximum number of concurrent requests. This can be used "
        "to help simulate an environment where a higher level component "
        "is enforcing a maximum number of concurrent requests. While the "
        "--request-rate argument controls the rate at which requests are "
        "initiated, this argument will control how many are actually allowed "
        "to execute at a time. This means that when used in combination, the "
        "actual request rate may be lower than specified with --request-rate, "
        "if the server is not processing requests fast enough to keep up.")

918
919
920
921
922
923
924
925
926
927
    parser.add_argument(
        "--model",
        type=str,
        required=True,
        help="Name of the model.",
    )
    parser.add_argument(
        "--tokenizer",
        type=str,
        help=
928
        "Name or path of the tokenizer, if not using the default tokenizer.",  # noqa: E501
929
930
931
932
933
934
935
936
    )
    parser.add_argument(
        "--best-of",
        type=int,
        default=1,
        help="Generates `best_of` sequences per prompt and "
        "returns the best one.",
    )
937
    parser.add_argument("--use-beam-search", action="store_true")
938
939
940
941
942
943
    parser.add_argument(
        "--num-prompts",
        type=int,
        default=1000,
        help="Number of prompts to process.",
    )
944
945
946
947
948
949
950
951
952
953
    parser.add_argument(
        "--logprobs",
        type=int,
        default=None,
        help=("Number of logprobs-per-token to compute & return as part of "
              "the request. If unspecified, then either (1) if beam search "
              "is disabled, no logprobs are computed & a single dummy "
              "logprob is returned for each token; or (2) if beam search "
              "is enabled 1 logprob per token is computed"),
    )
954
955
956
957
958
959
    parser.add_argument(
        "--request-rate",
        type=float,
        default=float("inf"),
        help="Number of requests per second. If this is inf, "
        "then all the requests are sent at time 0. "
960
961
962
963
964
965
966
967
968
969
970
971
972
973
        "Otherwise, we use Poisson process or gamma distribution "
        "to synthesize the request arrival times.",
    )
    parser.add_argument(
        "--burstiness",
        type=float,
        default=1.0,
        help="Burstiness factor of the request generation. "
        "Only take effect when request_rate is not inf. "
        "Default value is 1, which follows Poisson process. "
        "Otherwise, the request intervals follow a gamma distribution. "
        "A lower burstiness value (0 < burstiness < 1) results in more "
        "bursty requests. A higher burstiness value (burstiness > 1) "
        "results in a more uniform arrival of requests.",
974
    )
975
    parser.add_argument("--seed", type=int, default=0)
976
977
978
979
980
981
982
983
    parser.add_argument(
        "--trust-remote-code",
        action="store_true",
        help="Trust remote code from huggingface",
    )
    parser.add_argument(
        "--disable-tqdm",
        action="store_true",
984
        help="Specify to disable tqdm progress bar.",
985
986
    )
    parser.add_argument(
987
988
989
990
991
992
        "--profile",
        action="store_true",
        help="Use Torch Profiler. The endpoint must be launched with "
        "VLLM_TORCH_PROFILER_DIR to enable profiler.",
    )
    parser.add_argument(
993
994
995
996
        "--save-result",
        action="store_true",
        help="Specify to save benchmark results to a json file",
    )
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
    parser.add_argument(
        "--metadata",
        metavar="KEY=VALUE",
        nargs="*",
        help="Key-value pairs (e.g, --metadata version=0.3.3 tp=1) "
        "for metadata of this run to be saved in the result JSON file "
        "for record keeping purposes.",
    )
    parser.add_argument(
        "--result-dir",
        type=str,
        default=None,
        help="Specify directory to save benchmark json results."
        "If not specified, results are saved in the current directory.",
    )
1012
1013
1014
1015
1016
1017
1018
1019
1020
    parser.add_argument(
        "--result-filename",
        type=str,
        default=None,
        help="Specify the filename to save benchmark json results."
        "If not specified, results will be saved in "
        "{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
        " format.",
    )
1021
1022
1023
1024
1025
    parser.add_argument(
        "--ignore-eos",
        action="store_true",
        help="Set ignore_eos flag when sending the benchmark request."
        "Warning: ignore_eos is not supported in deepspeed_mii and tgi.")
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
    parser.add_argument(
        "--percentile-metrics",
        type=str,
        default="ttft,tpot,itl",
        help="Comma-seperated list of selected metrics to report percentils. "
        "This argument specifies the metrics to report percentiles. "
        "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". "
        "Default value is \"ttft,tpot,itl\".")
    parser.add_argument(
        "--metric-percentiles",
        type=str,
        default="99",
        help="Comma-seperated list of percentiles for selected metrics. "
        "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
        "Default value is \"99\". "
        "Use \"--percentile-metrics\" to select metrics.",
    )
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
    parser.add_argument(
        "--goodput",
        nargs="+",
        required=False,
        help="Specify service level objectives for goodput as \"KEY:VALUE\" "
        "pairs, where the key is a metric name, and the value is in "
        "milliseconds. Multiple \"KEY:VALUE\" pairs can be provided, "
        "separated by spaces. Allowed request level metric names are "
        "\"ttft\", \"tpot\", \"e2el\". For more context on the definition of "
        "goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 "
        "and the blog: https://hao-ai-lab.github.io/blogs/distserve")
1054

1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
    # group for dataset specific arguments
    sonnet_group = parser.add_argument_group("sonnet dataset options")
    sonnet_group.add_argument(
        "--sonnet-input-len",
        type=int,
        default=550,
        help=
        "Number of input tokens per request, used only for sonnet dataset.",
    )
    sonnet_group.add_argument(
        "--sonnet-output-len",
        type=int,
        default=150,
        help=
        "Number of output tokens per request, used only for sonnet dataset.",
    )
    sonnet_group.add_argument(
        "--sonnet-prefix-len",
        type=int,
        default=200,
        help=
        "Number of prefix tokens per request, used only for sonnet dataset.",
    )

    sharegpt_group = parser.add_argument_group("sharegpt dataset options")
    sharegpt_group.add_argument(
        "--sharegpt-output-len",
        type=int,
        default=None,
        help="Output length for each request. Overrides the output length "
        "from the ShareGPT dataset.")

    random_group = parser.add_argument_group("random dataset options")
    random_group.add_argument(
        "--random-input-len",
        type=int,
        default=1024,
        help=
        "Number of input tokens per request, used only for random sampling.",
    )
    random_group.add_argument(
        "--random-output-len",
        type=int,
        default=128,
        help=
        "Number of output tokens per request, used only for random sampling.",
    )
    random_group.add_argument(
        "--random-range-ratio",
        type=float,
        default=1.0,
        help="Range of sampled ratio of input/output length, "
        "used only for random sampling.",
    )
    random_group.add_argument(
        "--random-prefix-len",
        type=int,
        default=0,
        help="Number of fixed prefix tokens before random "
        " context. The length range of context in a random "
        " request is [random-prefix-len, "
        " random-prefix-len + random-prefix-len * random-range-ratio).")

    hf_group = parser.add_argument_group("hf dataset options")
    hf_group.add_argument("--hf-subset",
                          type=str,
                          default=None,
                          help="Subset of the HF dataset.")
    hf_group.add_argument("--hf-split",
                          type=str,
                          default=None,
                          help="Split of the HF dataset.")
    hf_group.add_argument(
        "--hf-output-len",
        type=int,
        default=None,
        help="Output length for each request. Overrides the output lengths "
        "from the sampled HF dataset.",
    )

1135
    args = parser.parse_args()
1136
    main(args)