benchmark_latency.py 4.01 KB
Newer Older
1
"""Benchmark the latency of processing a single batch of requests."""
2
3
4
5
6
import argparse
import time

import numpy as np
import torch
7
from tqdm import tqdm
8

Woosuk Kwon's avatar
Woosuk Kwon committed
9
from vllm import LLM, SamplingParams
10
11
12


def main(args: argparse.Namespace):
13
14
15
    print(args)

    # NOTE(woosuk): If the request cannot be processed in a single batch,
Zhuohan Li's avatar
Zhuohan Li committed
16
    # the engine will automatically process the request in multiple batches.
17
18
    llm = LLM(
        model=args.model,
19
        tokenizer=args.tokenizer,
20
        quantization=args.quantization,
21
22
        tensor_parallel_size=args.tensor_parallel_size,
        max_num_seqs=args.batch_size,
23
        trust_remote_code=args.trust_remote_code,
24
        dtype=args.dtype,
25
    )
26

Woosuk Kwon's avatar
Woosuk Kwon committed
27
28
29
30
31
    sampling_params = SamplingParams(
        n=args.n,
        temperature=0.0 if args.use_beam_search else 1.0,
        top_p=1.0,
        use_beam_search=args.use_beam_search,
32
        ignore_eos=True,
Woosuk Kwon's avatar
Woosuk Kwon committed
33
34
        max_tokens=args.output_len,
    )
35
    print(sampling_params)
36
    dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
37

38
    def run_to_completion(profile: bool = False):
39
        if profile:
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
            with torch.profiler.profile(activities=[
                    torch.profiler.ProfilerActivity.CPU,
                    torch.profiler.ProfilerActivity.CUDA,
            ]) as p:
                llm.generate(prompt_token_ids=dummy_prompt_token_ids,
                             sampling_params=sampling_params,
                             use_tqdm=False)
            print(p.key_averages())
        else:
            start_time = time.perf_counter()
            llm.generate(prompt_token_ids=dummy_prompt_token_ids,
                         sampling_params=sampling_params,
                         use_tqdm=False)
            end_time = time.perf_counter()
            latency = end_time - start_time
            return latency
56

57
58
    print("Warming up...")
    run_to_completion(profile=False)
59

60
61
62
63
64
    if args.profile:
        print("Profiling...")
        run_to_completion(profile=True)
        return

65
66
    # Benchmark.
    latencies = []
67
68
    for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
        latencies.append(run_to_completion(profile=False))
69
70
71
72
    print(f'Avg latency: {np.mean(latencies)} seconds')


if __name__ == '__main__':
73
    parser = argparse.ArgumentParser(
74
        description='Benchmark the latency of processing a single batch of '
75
        'requests till completion.')
76
    parser.add_argument('--model', type=str, default='facebook/opt-125m')
77
    parser.add_argument('--tokenizer', type=str, default=None)
78
79
    parser.add_argument('--quantization',
                        '-q',
chooper1's avatar
chooper1 committed
80
                        choices=['awq', 'squeezellm', None],
81
                        default=None)
82
    parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
83
84
85
    parser.add_argument('--input-len', type=int, default=32)
    parser.add_argument('--output-len', type=int, default=128)
    parser.add_argument('--batch-size', type=int, default=8)
86
87
88
    parser.add_argument('--n',
                        type=int,
                        default=1,
89
                        help='Number of generated sequences per prompt.')
90
    parser.add_argument('--use-beam-search', action='store_true')
91
92
93
    parser.add_argument('--num-iters',
                        type=int,
                        default=3,
94
                        help='Number of iterations to run.')
95
96
    parser.add_argument('--trust-remote-code',
                        action='store_true',
97
                        help='trust remote code from huggingface')
98
99
100
101
102
103
104
105
106
    parser.add_argument(
        '--dtype',
        type=str,
        default='auto',
        choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
        help='data type for model weights and activations. '
        'The "auto" option will use FP16 precision '
        'for FP32 and FP16 models, and BF16 precision '
        'for BF16 models.')
107
108
109
110
    parser.add_argument(
        '--profile',
        action='store_true',
        help='profile the generation process of a single batch')
111
112
    args = parser.parse_args()
    main(args)