benchmark_serving.py 26.5 KB
Newer Older
1
2
3
"""Benchmark online serving throughput.

On the server side, run one of the following commands:
4
    vLLM OpenAI API server
Ethan Xu's avatar
Ethan Xu committed
5
6
    vllm serve <your_model> \
        --swap-space 16 \
7
        --disable-log-requests
8
9

    (TGI backend)
Ronen Schaffer's avatar
Ronen Schaffer committed
10
    ./launch_tgi_server.sh <your_model> <max_batch_total_tokens>
11
12
13
14

On the client side, run:
    python benchmarks/benchmark_serving.py \
        --backend <backend> \
15
16
17
18
19
        --model <your_model> \
        --dataset-name sharegpt \
        --dataset-path <path to dataset> \
        --request-rate <request_rate> \ # By default <request_rate> is inf
        --num-prompts <num_prompts> # By default <num_prompts> is 1000
20

21
22
23
    when using tgi backend, add
        --endpoint /generate_stream
    to the end of the command above.
24
25
26
27
"""
import argparse
import asyncio
import json
28
import os
29
30
import random
import time
31
import warnings
32
33
from dataclasses import dataclass
from datetime import datetime
34
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
35
36

import numpy as np
37
38
from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
                                  RequestFuncOutput)
39
from tqdm.asyncio import tqdm
40
from transformers import PreTrainedTokenizerBase
41

42
43
44
45
try:
    from vllm.transformers_utils.tokenizer import get_tokenizer
except ImportError:
    from backend_request_func import get_tokenizer
46

47
48
49
50
51
try:
    from vllm.utils import FlexibleArgumentParser
except ImportError:
    from argparse import ArgumentParser as FlexibleArgumentParser

52
53
54
55
56
57
58
59
60
61
62

@dataclass
class BenchmarkMetrics:
    completed: int
    total_input: int
    total_output: int
    request_throughput: float
    input_throughput: float
    output_throughput: float
    mean_ttft_ms: float
    median_ttft_ms: float
63
    std_ttft_ms: float
64
65
66
    p99_ttft_ms: float
    mean_tpot_ms: float
    median_tpot_ms: float
67
    std_tpot_ms: float
68
    p99_tpot_ms: float
69
70
    mean_itl_ms: float
    median_itl_ms: float
71
    std_itl_ms: float
72
    p99_itl_ms: float
73
74


75
def sample_sharegpt_requests(
76
77
78
    dataset_path: str,
    num_requests: int,
    tokenizer: PreTrainedTokenizerBase,
79
    fixed_output_len: Optional[int] = None,
80
) -> List[Tuple[str, int, int]]:
81
82
    if fixed_output_len is not None and fixed_output_len < 4:
        raise ValueError("output_len too small")
83
84
85
86
    # Load the dataset.
    with open(dataset_path) as f:
        dataset = json.load(f)
    # Filter out the conversations with less than 2 turns.
87
    dataset = [data for data in dataset if len(data["conversations"]) >= 2]
88
    # Only keep the first two turns of each conversation.
89
90
    dataset = [(data["conversations"][0]["value"],
                data["conversations"][1]["value"]) for data in dataset]
91

92
93
    # Shuffle the dataset.
    random.shuffle(dataset)
94

95
    # Filter out sequences that are too long or too short
96
    filtered_dataset: List[Tuple[str, int, int]] = []
97
98
99
100
101
102
103
104
105
    for i in range(len(dataset)):
        if len(filtered_dataset) == num_requests:
            break

        # Tokenize the prompts and completions.
        prompt = dataset[i][0]
        prompt_token_ids = tokenizer(prompt).input_ids
        completion = dataset[i][1]
        completion_token_ids = tokenizer(completion).input_ids
106
        prompt_len = len(prompt_token_ids)
107
108
        output_len = len(completion_token_ids
                         ) if fixed_output_len is None else fixed_output_len
109
110
111
112
113
114
115
116
        if prompt_len < 4 or output_len < 4:
            # Prune too short sequences.
            continue
        if prompt_len > 1024 or prompt_len + output_len > 2048:
            # Prune too long sequences.
            continue
        filtered_dataset.append((prompt, prompt_len, output_len))

117
    return filtered_dataset
118
119


120
121
122
123
124
125
126
127
def sample_sonnet_requests(
    dataset_path: str,
    num_requests: int,
    input_len: int,
    output_len: int,
    prefix_len: int,
    tokenizer: PreTrainedTokenizerBase,
) -> List[Tuple[str, str, int, int]]:
128
129
130
    assert (
        input_len > prefix_len
    ), "'args.sonnet-input-len' must be greater than 'args.prefix-input-len'."
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

    # Load the dataset.
    with open(dataset_path) as f:
        poem_lines = f.readlines()

    # Tokenize the poem lines.
    poem_token_ids = tokenizer(poem_lines).input_ids
    average_poem_len = sum(
        len(token_ids) for token_ids in poem_token_ids) / len(poem_token_ids)

    # Base prefix for all requests.
    base_prompt = "Pick as many lines as you can from these poem lines:\n"
    base_message = [{
        "role": "user",
        "content": base_prompt,
    }]
    base_prompt_formatted = tokenizer.apply_chat_template(
        base_message, add_generation_prompt=True, tokenize=False)
    base_prompt_offset = len(tokenizer(base_prompt_formatted).input_ids)

151
152
153
    assert (
        input_len > base_prompt_offset
    ), f"Please set 'args.sonnet-input-len' higher than {base_prompt_offset}."
154
155
156
157
158
159
160
    num_input_lines = round(
        (input_len - base_prompt_offset) / average_poem_len)

    # First approximately `prefix_len` number of tokens in the
    # prompt are fixed poem lines.
    assert (
        prefix_len > base_prompt_offset
161
    ), f"Please set 'args.sonnet-prefix-len' higher than {base_prompt_offset}."
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189

    num_prefix_lines = round(
        (prefix_len - base_prompt_offset) / average_poem_len)
    prefix_lines = poem_lines[:num_prefix_lines]

    # Sample the rest of lines per request.
    sampled_requests: List[Tuple[str, int, int]] = []
    for _ in range(num_requests):
        sampled_lines = "".join(
            prefix_lines +
            random.sample(poem_lines, num_input_lines - num_prefix_lines))

        prompt = f"{base_prompt}{sampled_lines}"
        message = [
            {
                "role": "user",
                "content": prompt,
            },
        ]
        prompt_formatted = tokenizer.apply_chat_template(
            message, add_generation_prompt=True, tokenize=False)
        prompt_len = len(tokenizer(prompt_formatted).input_ids)
        sampled_requests.append(
            (prompt, prompt_formatted, prompt_len, output_len))

    return sampled_requests


190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
def sample_random_requests(
        input_len: int, output_len: int, num_prompts: int, range_ratio: float,
        tokenizer: PreTrainedTokenizerBase) -> List[Tuple[str, int, int]]:

    input_lens = np.random.randint(
        int(input_len * range_ratio),
        input_len + 1,
        size=num_prompts,
    )
    output_lens = np.random.randint(
        int(output_len * range_ratio),
        output_len + 1,
        size=num_prompts,
    )
    offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
    input_requests = []
206
    for i in range(num_prompts):
207
208
209
210
211
212
213
214
        prompt = tokenizer.decode([(offsets[i] + i + j) % tokenizer.vocab_size
                                   for j in range(input_lens[i])])
        input_requests.append(
            (prompt, int(input_lens[i]), int(output_lens[i])))

    return input_requests


215
216
217
218
219
220
221
222
223
224
225
async def get_request(
    input_requests: List[Tuple[str, int, int]],
    request_rate: float,
) -> AsyncGenerator[Tuple[str, int, int], None]:
    input_requests = iter(input_requests)
    for request in input_requests:
        yield request

        if request_rate == float("inf"):
            # If the request rate is infinity, then we don't need to wait.
            continue
226

227
228
229
230
231
232
        # Sample the request interval from the exponential distribution.
        interval = np.random.exponential(1.0 / request_rate)
        # The next request will be sent after the interval.
        await asyncio.sleep(interval)


233
234
235
236
237
def calculate_metrics(
    input_requests: List[Tuple[str, int, int]],
    outputs: List[RequestFuncOutput],
    dur_s: float,
    tokenizer: PreTrainedTokenizerBase,
238
) -> Tuple[BenchmarkMetrics, List[int]]:
239
    actual_output_lens: List[int] = []
240
241
    total_input = 0
    completed = 0
242
243
244
    itls: List[float] = []
    tpots: List[float] = []
    ttfts: List[float] = []
245
246
    for i in range(len(outputs)):
        if outputs[i].success:
247
248
249
            # We use the tokenizer to count the number of output tokens for all
            # serving backends instead of looking at len(outputs[i].itl) since
            # multiple output tokens may be bundled together
250
            # Note : this may inflate the output token count slightly
251
252
253
            output_len = len(
                tokenizer(outputs[i].generated_text,
                          add_special_tokens=False).input_ids)
254
            actual_output_lens.append(output_len)
255
            total_input += input_requests[i][1]
256
257
258
            if output_len > 1:
                tpots.append(
                    (outputs[i].latency - outputs[i].ttft) / (output_len - 1))
259
            itls += outputs[i].itl
260
261
            ttfts.append(outputs[i].ttft)
            completed += 1
262
263
        else:
            actual_output_lens.append(0)
264

265
266
267
268
269
    if completed == 0:
        warnings.warn(
            "All requests failed. This is likely due to a misconfiguration "
            "on the benchmark arguments.",
            stacklevel=2)
270
271
272
    metrics = BenchmarkMetrics(
        completed=completed,
        total_input=total_input,
273
        total_output=sum(actual_output_lens),
274
275
        request_throughput=completed / dur_s,
        input_throughput=total_input / dur_s,
276
277
278
279
        output_throughput=sum(actual_output_lens) / dur_s,
        mean_ttft_ms=np.mean(ttfts or 0) *
        1000,  # ttfts is empty if streaming is not supported by backend
        median_ttft_ms=np.median(ttfts or 0) * 1000,
280
        std_ttft_ms=np.std(ttfts or 0) * 1000,
281
        p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000,
282
283
        mean_tpot_ms=np.mean(tpots or 0) * 1000,
        median_tpot_ms=np.median(tpots or 0) * 1000,
284
        std_tpot_ms=np.std(tpots or 0) * 1000,
285
        p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000,
286
287
        mean_itl_ms=np.mean(itls or 0) * 1000,
        median_itl_ms=np.median(itls or 0) * 1000,
288
        std_itl_ms=np.std(itls or 0) * 1000,
289
        p99_itl_ms=np.percentile(itls or 0, 99) * 1000,
290
    )
291

292
    return metrics, actual_output_lens
293

294
295
296
297

async def benchmark(
    backend: str,
    api_url: str,
298
    base_url: str,
299
300
    model_id: str,
    tokenizer: PreTrainedTokenizerBase,
301
302
303
304
    input_requests: List[Tuple[str, int, int]],
    best_of: int,
    use_beam_search: bool,
    request_rate: float,
305
    disable_tqdm: bool,
306
    profile: bool,
307
308
):
    if backend in ASYNC_REQUEST_FUNCS:
309
        request_func = ASYNC_REQUEST_FUNCS[backend]
310
311
312
    else:
        raise ValueError(f"Unknown backend: {backend}")

313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
    print("Starting initial single prompt test run...")
    test_prompt, test_prompt_len, test_output_len = input_requests[0]
    test_input = RequestFuncInput(
        model=model_id,
        prompt=test_prompt,
        api_url=api_url,
        prompt_len=test_prompt_len,
        output_len=test_output_len,
        best_of=best_of,
        use_beam_search=use_beam_search,
    )
    test_output = await request_func(request_func_input=test_input)
    if not test_output.success:
        raise ValueError(
            "Initial test run failed - Please make sure benchmark arguments "
            f"are correctly specified. Error: {test_output.error}")
    else:
        print("Initial test run completed. Starting main benchmark run...")
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346

    if profile:
        print("Starting profiler...")
        profile_input = RequestFuncInput(
            model=model_id,
            prompt=test_prompt,
            api_url=base_url + "/start_profile",
            prompt_len=test_prompt_len,
            output_len=test_output_len,
            best_of=best_of,
            use_beam_search=use_beam_search,
        )
        profile_output = await request_func(request_func_input=profile_input)
        if profile_output.success:
            print("Profiler started")

347
348
    print(f"Traffic request rate: {request_rate}")

349
350
    pbar = None if disable_tqdm else tqdm(total=len(input_requests))

351
    benchmark_start_time = time.perf_counter()
352
    tasks: List[asyncio.Task] = []
353
354
    async for request in get_request(input_requests, request_rate):
        prompt, prompt_len, output_len = request
355
356
357
358
359
360
361
362
363
364
365
366
367
        request_func_input = RequestFuncInput(
            model=model_id,
            prompt=prompt,
            api_url=api_url,
            prompt_len=prompt_len,
            output_len=output_len,
            best_of=best_of,
            use_beam_search=use_beam_search,
        )
        tasks.append(
            asyncio.create_task(
                request_func(request_func_input=request_func_input,
                             pbar=pbar)))
368
    outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
369

370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
    if profile:
        print("Stopping profiler...")
        profile_input = RequestFuncInput(
            model=model_id,
            prompt=test_prompt,
            api_url=base_url + "/stop_profile",
            prompt_len=test_prompt_len,
            output_len=test_output_len,
            best_of=best_of,
            use_beam_search=use_beam_search,
        )
        profile_output = await request_func(request_func_input=profile_input)
        if profile_output.success:
            print("Profiler stopped")

385
    if pbar is not None:
386
387
388
389
        pbar.close()

    benchmark_duration = time.perf_counter() - benchmark_start_time

390
    metrics, actual_output_lens = calculate_metrics(
391
392
393
394
395
396
        input_requests=input_requests,
        outputs=outputs,
        dur_s=benchmark_duration,
        tokenizer=tokenizer,
    )

397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
    print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
    print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
    print("{:<40} {:<10.2f}".format("Benchmark duration (s):",
                                    benchmark_duration))
    print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
    print("{:<40} {:<10}".format("Total generated tokens:",
                                 metrics.total_output))
    print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
                                    metrics.request_throughput))
    print("{:<40} {:<10.2f}".format("Input token throughput (tok/s):",
                                    metrics.input_throughput))
    print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
                                    metrics.output_throughput))
    print("{s:{c}^{n}}".format(s='Time to First Token', n=50, c='-'))
    print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms))
    print("{:<40} {:<10.2f}".format("Median TTFT (ms):",
                                    metrics.median_ttft_ms))
    print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms))
    print("{s:{c}^{n}}".format(s='Time per Output Token (excl. 1st token)',
                               n=50,
                               c='-'))
    print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms))
    print("{:<40} {:<10.2f}".format("Median TPOT (ms):",
                                    metrics.median_tpot_ms))
    print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms))
422
423
424
425
    print("{s:{c}^{n}}".format(s='Inter-token Latency', n=50, c='-'))
    print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms))
    print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms))
    print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms))
426
    print("=" * 50)
427
428
429
430
431
432

    result = {
        "duration": benchmark_duration,
        "completed": metrics.completed,
        "total_input_tokens": metrics.total_input,
        "total_output_tokens": metrics.total_output,
433
        "request_throughput": metrics.request_throughput,
434
435
436
437
        "input_throughput": metrics.input_throughput,
        "output_throughput": metrics.output_throughput,
        "mean_ttft_ms": metrics.mean_ttft_ms,
        "median_ttft_ms": metrics.median_ttft_ms,
438
        "std_ttft_ms": metrics.std_ttft_ms,
439
440
441
        "p99_ttft_ms": metrics.p99_ttft_ms,
        "mean_tpot_ms": metrics.mean_tpot_ms,
        "median_tpot_ms": metrics.median_tpot_ms,
442
        "std_tpot_ms": metrics.std_tpot_ms,
443
        "p99_tpot_ms": metrics.p99_tpot_ms,
444
445
        "mean_itl_ms": metrics.mean_itl_ms,
        "median_itl_ms": metrics.median_itl_ms,
446
        "std_itl_ms": metrics.std_itl_ms,
447
        "p99_itl_ms": metrics.p99_itl_ms,
448
449
450
451
452
453
        "input_lens": [output.prompt_len for output in outputs],
        "output_lens": actual_output_lens,
        "ttfts": [output.ttft for output in outputs],
        "itls": [output.itl for output in outputs],
        "generated_texts": [output.generated_text for output in outputs],
        "errors": [output.error for output in outputs],
454
455
    }
    return result
456
457
458
459
460
461
462


def main(args: argparse.Namespace):
    print(args)
    random.seed(args.seed)
    np.random.seed(args.seed)

463
464
465
466
467
468
    backend = args.backend
    model_id = args.model
    tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model

    if args.base_url is not None:
        api_url = f"{args.base_url}{args.endpoint}"
469
        base_url = f"{args.base_url}"
470
471
    else:
        api_url = f"http://{args.host}:{args.port}{args.endpoint}"
472
        base_url = f"http://{args.host}:{args.port}"
473
474

    tokenizer = get_tokenizer(tokenizer_id,
475
                              trust_remote_code=args.trust_remote_code)
476
477
478
479
480
481
482
483
484
485
486

    if args.dataset is not None:
        warnings.warn(
            "The '--dataset' argument will be deprecated in the next "
            "release. Please use '--dataset-name' and "
            "'--dataset-path' in the future runs.",
            stacklevel=2)
        input_requests = sample_sharegpt_requests(
            dataset_path=args.dataset,
            num_requests=args.num_prompts,
            tokenizer=tokenizer,
487
            fixed_output_len=args.sharegpt_output_len,
488
489
490
491
492
493
494
        )

    elif args.dataset_name == "sharegpt":
        input_requests = sample_sharegpt_requests(
            dataset_path=args.dataset_path,
            num_requests=args.num_prompts,
            tokenizer=tokenizer,
495
            fixed_output_len=args.sharegpt_output_len,
496
497
498
499
500
501
502
503
        )

    elif args.dataset_name == "sonnet":
        # Do not format the prompt, pass to message directly
        if args.backend == "openai-chat":
            input_requests = sample_sonnet_requests(
                dataset_path=args.dataset_path,
                num_requests=args.num_prompts,
504
505
506
                input_len=args.sonnet_input_len,
                output_len=args.sonnet_output_len,
                prefix_len=args.sonnet_prefix_len,
507
508
509
510
511
512
513
514
515
516
517
518
                tokenizer=tokenizer,
            )
            input_requests = [(prompt, prompt_len, output_len)
                              for prompt, prompt_formatted, prompt_len,
                              output_len in input_requests]
        else:
            assert (
                tokenizer.chat_template or tokenizer.default_chat_template
            ), "Tokenizer/model must have chat template for sonnet dataset."
            input_requests = sample_sonnet_requests(
                dataset_path=args.dataset_path,
                num_requests=args.num_prompts,
519
520
521
                input_len=args.sonnet_input_len,
                output_len=args.sonnet_output_len,
                prefix_len=args.sonnet_prefix_len,
522
523
524
525
526
527
                tokenizer=tokenizer,
            )
            input_requests = [(prompt_formatted, prompt_len, output_len)
                              for prompt, prompt_formatted, prompt_len,
                              output_len in input_requests]

528
529
    elif args.dataset_name == "random":
        input_requests = sample_random_requests(
530
531
            input_len=args.random_input_len,
            output_len=args.random_output_len,
532
            num_prompts=args.num_prompts,
533
            range_ratio=args.random_range_ratio,
534
535
536
            tokenizer=tokenizer,
        )

537
538
    else:
        raise ValueError(f"Unknown dataset: {args.dataset_name}")
539

540
541
542
543
    benchmark_result = asyncio.run(
        benchmark(
            backend=backend,
            api_url=api_url,
544
            base_url=base_url,
545
546
547
548
549
550
551
            model_id=model_id,
            tokenizer=tokenizer,
            input_requests=input_requests,
            best_of=args.best_of,
            use_beam_search=args.use_beam_search,
            request_rate=args.request_rate,
            disable_tqdm=args.disable_tqdm,
552
            profile=args.profile,
553
554
555
556
        ))

    # Save config and results to json
    if args.save_result:
557
        result_json: Dict[str, Any] = {}
558
559
560
561
562
563
564
565
566
567
568

        # Setup
        current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
        result_json["date"] = current_dt
        result_json["backend"] = backend
        result_json["model_id"] = model_id
        result_json["tokenizer_id"] = tokenizer_id
        result_json["best_of"] = args.best_of
        result_json["use_beam_search"] = args.use_beam_search
        result_json["num_prompts"] = args.num_prompts

569
570
571
572
573
574
575
576
577
578
579
        # Metadata
        if args.metadata:
            for item in args.metadata:
                if "=" in item:
                    kvstring = item.split("=")
                    result_json[kvstring[0].strip()] = kvstring[1].strip()
                else:
                    raise ValueError(
                        "Invalid metadata format. Please use KEY=VALUE format."
                    )

580
581
582
583
584
585
586
587
588
        # Traffic
        result_json["request_rate"] = (
            args.request_rate if args.request_rate < float("inf") else "inf")

        # Merge with benchmark result
        result_json = {**result_json, **benchmark_result}

        # Save to file
        base_model_id = model_id.split("/")[-1]
589
        file_name = f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"  #noqa
590
591
        if args.result_filename:
            file_name = args.result_filename
592
593
        if args.result_dir:
            file_name = os.path.join(args.result_dir, file_name)
594
595
        with open(file_name, "w") as outfile:
            json.dump(result_json, outfile)
596
597
598


if __name__ == "__main__":
599
    parser = FlexibleArgumentParser(
600
        description="Benchmark the online serving throughput.")
601
602
603
604
605
606
607
608
609
610
611
612
    parser.add_argument(
        "--backend",
        type=str,
        default="vllm",
        choices=list(ASYNC_REQUEST_FUNCS.keys()),
    )
    parser.add_argument(
        "--base-url",
        type=str,
        default=None,
        help="Server or API base url if not using http host and port.",
    )
613
    parser.add_argument("--host", type=str, default="localhost")
614
    parser.add_argument("--port", type=int, default=8000)
615
616
617
    parser.add_argument(
        "--endpoint",
        type=str,
618
        default="/v1/completions",
619
620
        help="API endpoint.",
    )
621
622
623
624
625
626
627
628
629
630
631
    parser.add_argument(
        "--dataset",
        type=str,
        default=None,
        help="Path to the ShareGPT dataset, will be deprecated in the "
        "next release.",
    )
    parser.add_argument(
        "--dataset-name",
        type=str,
        default="sharegpt",
632
        choices=["sharegpt", "sonnet", "random"],
633
634
635
        help="Name of the dataset to benchmark on.",
    )
    parser.add_argument("--dataset-path",
636
                        type=str,
637
                        default=None,
638
                        help="Path to the dataset.")
639
640
641
642
643
644
645
646
647
648
    parser.add_argument(
        "--model",
        type=str,
        required=True,
        help="Name of the model.",
    )
    parser.add_argument(
        "--tokenizer",
        type=str,
        help=
649
        "Name or path of the tokenizer, if not using the default tokenizer.",  # noqa: E501
650
651
652
653
654
655
656
657
    )
    parser.add_argument(
        "--best-of",
        type=int,
        default=1,
        help="Generates `best_of` sequences per prompt and "
        "returns the best one.",
    )
658
    parser.add_argument("--use-beam-search", action="store_true")
659
660
661
662
663
664
    parser.add_argument(
        "--num-prompts",
        type=int,
        default=1000,
        help="Number of prompts to process.",
    )
665
666
667
668
669
670
    parser.add_argument(
        "--sharegpt-output-len",
        type=int,
        default=None,
        help="Output length for each request. Overrides the output length "
        "from the ShareGPT dataset.")
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
    parser.add_argument(
        "--sonnet-input-len",
        type=int,
        default=550,
        help=
        "Number of input tokens per request, used only for sonnet dataset.",
    )
    parser.add_argument(
        "--sonnet-output-len",
        type=int,
        default=150,
        help=
        "Number of output tokens per request, used only for sonnet dataset.",
    )
    parser.add_argument(
        "--sonnet-prefix-len",
        type=int,
        default=200,
        help=
        "Number of prefix tokens per request, used only for sonnet dataset.",
    )
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
    parser.add_argument(
        "--random-input-len",
        type=int,
        default=1024,
        help=
        "Number of input tokens per request, used only for random sampling.",
    )
    parser.add_argument(
        "--random-output-len",
        type=int,
        default=128,
        help=
        "Number of output tokens per request, used only for random sampling.",
    )
    parser.add_argument(
        "--random-range-ratio",
        type=float,
        default=1.0,
        help="Range of sampled ratio of input/output length, "
        "used only for random sampling.",
    )
713
714
715
716
717
718
719
720
721
    parser.add_argument(
        "--request-rate",
        type=float,
        default=float("inf"),
        help="Number of requests per second. If this is inf, "
        "then all the requests are sent at time 0. "
        "Otherwise, we use Poisson process to synthesize "
        "the request arrival times.",
    )
722
    parser.add_argument("--seed", type=int, default=0)
723
724
725
726
727
728
729
730
    parser.add_argument(
        "--trust-remote-code",
        action="store_true",
        help="Trust remote code from huggingface",
    )
    parser.add_argument(
        "--disable-tqdm",
        action="store_true",
731
        help="Specify to disable tqdm progress bar.",
732
733
    )
    parser.add_argument(
734
735
736
737
738
739
        "--profile",
        action="store_true",
        help="Use Torch Profiler. The endpoint must be launched with "
        "VLLM_TORCH_PROFILER_DIR to enable profiler.",
    )
    parser.add_argument(
740
741
742
743
        "--save-result",
        action="store_true",
        help="Specify to save benchmark results to a json file",
    )
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
    parser.add_argument(
        "--metadata",
        metavar="KEY=VALUE",
        nargs="*",
        help="Key-value pairs (e.g, --metadata version=0.3.3 tp=1) "
        "for metadata of this run to be saved in the result JSON file "
        "for record keeping purposes.",
    )
    parser.add_argument(
        "--result-dir",
        type=str,
        default=None,
        help="Specify directory to save benchmark json results."
        "If not specified, results are saved in the current directory.",
    )
759
760
761
762
763
764
765
766
767
    parser.add_argument(
        "--result-filename",
        type=str,
        default=None,
        help="Specify the filename to save benchmark json results."
        "If not specified, results will be saved in "
        "{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
        " format.",
    )
768

769
770
    args = parser.parse_args()
    main(args)