test_metrics.py 14.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
5
6
7
import subprocess
import sys
import tempfile
import time
8
9
10
11
from http import HTTPStatus

import openai
import pytest
12
import pytest_asyncio
13
14
15
16
import requests
from prometheus_client.parser import text_string_to_metric_families
from transformers import AutoTokenizer

17
18
from vllm import version

19
20
from ...utils import RemoteOpenAIServer

21
22
23
24
MODELS = {
    "text": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
    "multimodal": "HuggingFaceTB/SmolVLM-256M-Instruct",
}
25
PREV_MINOR_VERSION = version._prev_minor_version()
26
27


28
29
30
31
32
@pytest.fixture(scope="module", params=list(MODELS.keys()))
def model_key(request):
    yield request.param


33
34
35
36
37
38
39
40
41
42
43
44
45
46
@pytest.fixture(scope="module")
def default_server_args():
    return [
        # use half precision for speed and memory savings in CI environment
        "--dtype",
        "bfloat16",
        "--max-model-len",
        "1024",
        "--enforce-eager",
        "--max-num-seqs",
        "128",
    ]


47
48
49
50
51
52
53
54
55
@pytest.fixture(
    scope="module",
    params=[
        "",
        "--enable-chunked-prefill",
        "--disable-frontend-multiprocessing",
        f"--show-hidden-metrics-for-version={PREV_MINOR_VERSION}",
    ],
)
56
def server(model_key, default_server_args, request):
57
58
    if request.param:
        default_server_args.append(request.param)
59

60
61
    model_name = MODELS[model_key]
    with RemoteOpenAIServer(model_name, default_server_args) as remote_server:
62
63
64
65
66
67
68
        yield remote_server


@pytest_asyncio.fixture
async def client(server):
    async with server.get_async_client() as cl:
        yield cl
69
70
71


_PROMPT = "Hello my name is Robert and I love magic"
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"


def _get_expected_values(num_requests: int, prompt_ids: list[int], max_tokens: int):
    num_prompt_tokens = len(prompt_ids)

    # {metric_family: [(suffix, expected_value)]}
    return {
        "vllm:time_to_first_token_seconds": [("_count", num_requests)],
        "vllm:time_per_output_token_seconds": [
            ("_count", num_requests * (max_tokens - 1))
        ],
        "vllm:e2e_request_latency_seconds": [("_count", num_requests)],
        "vllm:request_queue_time_seconds": [("_count", num_requests)],
        "vllm:request_inference_time_seconds": [("_count", num_requests)],
        "vllm:request_prefill_time_seconds": [("_count", num_requests)],
        "vllm:request_decode_time_seconds": [("_count", num_requests)],
        "vllm:request_prompt_tokens": [
            ("_sum", num_requests * num_prompt_tokens),
            ("_count", num_requests),
        ],
        "vllm:request_generation_tokens": [
            ("_sum", num_requests * max_tokens),
            ("_count", num_requests),
        ],
        "vllm:request_params_n": [("_count", num_requests)],
        "vllm:request_params_max_tokens": [
            ("_sum", num_requests * max_tokens),
            ("_count", num_requests),
        ],
        "vllm:iteration_tokens_total": [
            (
                "_sum",
                num_requests * (num_prompt_tokens + max_tokens),
            ),
            ("_count", num_requests * max_tokens),
        ],
        "vllm:prompt_tokens": [("_total", num_requests * num_prompt_tokens)],
        "vllm:generation_tokens": [("_total", num_requests * max_tokens)],
        "vllm:request_success": [("_total", num_requests)],
    }
113
114
115


@pytest.mark.asyncio
116
async def test_metrics_counts(
117
118
    server: RemoteOpenAIServer,
    client: openai.AsyncClient,
119
    model_key: str,
120
):
121
122
123
124
125
126
127
128
129
130
    if model_key == "multimodal":
        pytest.skip("Unnecessary test")

    model_name = MODELS[model_key]
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    prompt_ids = tokenizer.encode(_PROMPT)
    num_requests = 10
    max_tokens = 10

    for _ in range(num_requests):
131
132
        # sending a request triggers the metrics to be logged.
        await client.completions.create(
133
134
135
            model=model_name,
            prompt=prompt_ids,
            max_tokens=max_tokens,
136
        )
137

138
    response = requests.get(server.url_for("metrics"))
139
140
141
142
    print(response.text)
    assert response.status_code == HTTPStatus.OK

    # Loop over all expected metric_families
143
144
145
    expected_values = _get_expected_values(num_requests, prompt_ids, max_tokens)
    for metric_family, suffix_values_list in expected_values.items():
        if metric_family not in EXPECTED_METRICS_V1 or (
146
147
148
            not server.show_hidden_metrics
            and metric_family in HIDDEN_DEPRECATED_METRICS
        ):
149
150
            continue

151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
        found_metric = False

        # Check to see if the metric_family is found in the prom endpoint.
        for family in text_string_to_metric_families(response.text):
            if family.name == metric_family:
                found_metric = True

                # Check that each suffix is found in the prom endpoint.
                for suffix, expected_value in suffix_values_list:
                    metric_name_w_suffix = f"{metric_family}{suffix}"
                    found_suffix = False

                    for sample in family.samples:
                        if sample.name == metric_name_w_suffix:
                            found_suffix = True

                            # For each suffix, value sure the value matches
                            # what we expect.
                            assert sample.value == expected_value, (
                                f"{metric_name_w_suffix} expected value of "
                                f"{expected_value} did not match found value "
172
173
                                f"{sample.value}"
                            )
174
175
176
177
178
179
                            break
                    assert found_suffix, (
                        f"Did not find {metric_name_w_suffix} in prom endpoint"
                    )
                break

180
        assert found_metric, f"Did not find {metric_family} in prom endpoint"
181
182


183
184
185
EXPECTED_METRICS_V1 = [
    "vllm:num_requests_running",
    "vllm:num_requests_waiting",
186
    "vllm:gpu_cache_usage_perc",
187
188
    "vllm:gpu_prefix_cache_queries",
    "vllm:gpu_prefix_cache_hits",
189
190
191
    "vllm:kv_cache_usage_perc",
    "vllm:prefix_cache_queries",
    "vllm:prefix_cache_hits",
192
    "vllm:num_preemptions_total",
193
194
    "vllm:prompt_tokens_total",
    "vllm:generation_tokens_total",
195
    "vllm:iteration_tokens_total",
196
    "vllm:cache_config_info",
197
    "vllm:request_success_total",
198
199
200
201
202
203
    "vllm:request_prompt_tokens_sum",
    "vllm:request_prompt_tokens_bucket",
    "vllm:request_prompt_tokens_count",
    "vllm:request_generation_tokens_sum",
    "vllm:request_generation_tokens_bucket",
    "vllm:request_generation_tokens_count",
204
205
206
207
208
209
    "vllm:request_params_n_sum",
    "vllm:request_params_n_bucket",
    "vllm:request_params_n_count",
    "vllm:request_params_max_tokens_sum",
    "vllm:request_params_max_tokens_bucket",
    "vllm:request_params_max_tokens_count",
210
211
212
    "vllm:time_per_output_token_seconds_sum",
    "vllm:time_per_output_token_seconds_bucket",
    "vllm:time_per_output_token_seconds_count",
213
214
215
216
217
218
    "vllm:time_to_first_token_seconds_sum",
    "vllm:time_to_first_token_seconds_bucket",
    "vllm:time_to_first_token_seconds_count",
    "vllm:inter_token_latency_seconds_sum",
    "vllm:inter_token_latency_seconds_bucket",
    "vllm:inter_token_latency_seconds_count",
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
    "vllm:e2e_request_latency_seconds_sum",
    "vllm:e2e_request_latency_seconds_bucket",
    "vllm:e2e_request_latency_seconds_count",
    "vllm:request_queue_time_seconds_sum",
    "vllm:request_queue_time_seconds_bucket",
    "vllm:request_queue_time_seconds_count",
    "vllm:request_inference_time_seconds_sum",
    "vllm:request_inference_time_seconds_bucket",
    "vllm:request_inference_time_seconds_count",
    "vllm:request_prefill_time_seconds_sum",
    "vllm:request_prefill_time_seconds_bucket",
    "vllm:request_prefill_time_seconds_count",
    "vllm:request_decode_time_seconds_sum",
    "vllm:request_decode_time_seconds_bucket",
    "vllm:request_decode_time_seconds_count",
234
235
]

236
237
238
239
240
EXPECTED_METRICS_MM = [
    "vllm:mm_cache_queries",
    "vllm:mm_cache_hits",
]

241
HIDDEN_DEPRECATED_METRICS: list[str] = [
242
243
244
    "vllm:gpu_cache_usage_perc",
    "vllm:gpu_prefix_cache_queries",
    "vllm:gpu_prefix_cache_hits",
245
246
247
248
    "vllm:time_per_output_token_seconds_sum",
    "vllm:time_per_output_token_seconds_bucket",
    "vllm:time_per_output_token_seconds_count",
]
249

250
251

@pytest.mark.asyncio
252
async def test_metrics_exist(
253
254
    server: RemoteOpenAIServer,
    client: openai.AsyncClient,
255
    model_key: str,
256
):
257
258
    model_name = MODELS[model_key]

259
    # sending a request triggers the metrics to be logged.
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
    if model_key == "text":
        await client.completions.create(
            model=model_name,
            prompt="Hello, my name is",
            max_tokens=5,
            temperature=0.0,
        )
    else:
        await client.chat.completions.create(
            model=model_name,
            messages=[
                {
                    "role": "user",
                    "content": [
                        {"type": "image_url", "image_url": {"url": _IMAGE_URL}},
                        {"type": "text", "text": "What's in this image?"},
                    ],
                }
            ],
            max_tokens=5,
            temperature=0.0,
        )
282

283
    response = requests.get(server.url_for("metrics"))
284
285
    assert response.status_code == HTTPStatus.OK

286
287
288
289
290
291
    expected_metrics = EXPECTED_METRICS_V1
    if model_key == "multimodal":
        # NOTE: Don't use in-place assignment
        expected_metrics = expected_metrics + EXPECTED_METRICS_MM

    for metric in expected_metrics:
292
        if metric in HIDDEN_DEPRECATED_METRICS and not server.show_hidden_metrics:
293
294
            continue
        assert metric in response.text
295
296


297
@pytest.mark.asyncio
298
async def test_abort_metrics_reset(
299
300
    server: RemoteOpenAIServer,
    client: openai.AsyncClient,
301
    model_key: str,
302
):
303
304
305
306
    model_name = MODELS[model_key]
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    prompt_ids = tokenizer.encode(_PROMPT)

307
    running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
308
        server,
309
    )
310
311
312
313
314
315
316
317
318
319
320

    # Expect no running requests or kvcache usage
    assert running_requests == 0
    assert waiting_requests == 0
    assert kv_cache_usage == 0.0

    # Start some long-running requests that we can abort
    tasks = []
    for _ in range(3):
        task = asyncio.create_task(
            client.completions.create(
321
322
                model=model_name,
                prompt=prompt_ids,
323
                max_tokens=100,  # Long generation to give time to abort
324
325
326
                temperature=0.0,
            )
        )
327
328
329
330
331
332
        tasks.append(task)

    # Wait a bit for requests to start processing
    await asyncio.sleep(0.5)

    # Check that we have running requests
333
    running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
334
        server,
335
    )
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353

    # Expect running requests and kvcache usage
    assert running_requests > 0
    assert kv_cache_usage > 0

    # Cancel all tasks to abort the requests
    for task in tasks:
        task.cancel()

    # Wait for cancellations to be processed
    await asyncio.sleep(1.0)

    # Check that metrics have reset to zero
    response = requests.get(server.url_for("metrics"))
    assert response.status_code == HTTPStatus.OK

    # Verify running and waiting requests counts and KV cache usage are zero
    running_requests_after, waiting_requests_after, kv_cache_usage_after = (
354
        _get_running_metrics_from_api(server)
355
    )
356

357
358
359
360
361
362
363
364
365
    assert running_requests_after == 0, (
        f"Expected 0 running requests after abort, got {running_requests_after}"
    )
    assert waiting_requests_after == 0, (
        f"Expected 0 waiting requests after abort, got {waiting_requests_after}"
    )
    assert kv_cache_usage_after == 0, (
        f"Expected 0% KV cache usage after abort, got {kv_cache_usage_after}"
    )
366
367


368
def _get_running_metrics_from_api(server: RemoteOpenAIServer):
369
370
371
372
373
374
375
376
    """Return (running_count, waiting_count, kv_cache_usage)"""

    response = requests.get(server.url_for("metrics"))
    assert response.status_code == HTTPStatus.OK

    # Verify running and waiting requests counts and KV cache usage are zero
    running_requests, waiting_requests, kv_cache_usage = None, None, None

377
    kv_cache_usage_metric = "vllm:kv_cache_usage_perc"
378

379
380
381
382
383
384
385
386
387
388
389
    for family in text_string_to_metric_families(response.text):
        if family.name == "vllm:num_requests_running":
            for sample in family.samples:
                if sample.name == "vllm:num_requests_running":
                    running_requests = sample.value
                    break
        elif family.name == "vllm:num_requests_waiting":
            for sample in family.samples:
                if sample.name == "vllm:num_requests_waiting":
                    waiting_requests = sample.value
                    break
390
        elif family.name == kv_cache_usage_metric:
391
            for sample in family.samples:
392
                if sample.name == kv_cache_usage_metric:
393
394
395
396
397
398
399
400
401
402
                    kv_cache_usage = sample.value
                    break

    assert running_requests is not None
    assert waiting_requests is not None
    assert kv_cache_usage is not None

    return running_requests, waiting_requests, kv_cache_usage


403
def test_metrics_exist_run_batch():
404
    input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "You are a helpful assistant."}}"""  # noqa: E501
405
406
407
408
409

    base_url = "0.0.0.0"
    port = "8001"
    server_url = f"http://{base_url}:{port}"

410
411
412
413
    with (
        tempfile.NamedTemporaryFile("w") as input_file,
        tempfile.NamedTemporaryFile("r") as output_file,
    ):
414
415
        input_file.write(input_batch)
        input_file.flush()
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
        proc = subprocess.Popen(
            [
                sys.executable,
                "-m",
                "vllm.entrypoints.openai.run_batch",
                "-i",
                input_file.name,
                "-o",
                output_file.name,
                "--model",
                "intfloat/multilingual-e5-small",
                "--enable-metrics",
                "--url",
                base_url,
                "--port",
                port,
            ],
        )
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448

        def is_server_up(url):
            try:
                response = requests.get(url)
                return response.status_code == 200
            except requests.ConnectionError:
                return False

        while not is_server_up(server_url):
            time.sleep(1)

        response = requests.get(server_url + "/metrics")
        assert response.status_code == HTTPStatus.OK

        proc.wait()