test_metrics.py 14.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
5
6
7
import subprocess
import sys
import tempfile
import time
8
9
10
11
from http import HTTPStatus

import openai
import pytest
12
import pytest_asyncio
13
14
15
16
import requests
from prometheus_client.parser import text_string_to_metric_families
from transformers import AutoTokenizer

17
18
from tests.conftest import LocalAssetServer
from tests.utils import RemoteOpenAIServer
19
20
from vllm import version

21
22
23
24
MODELS = {
    "text": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
    "multimodal": "HuggingFaceTB/SmolVLM-256M-Instruct",
}
25
PREV_MINOR_VERSION = version._prev_minor_version()
26
27


28
29
30
31
32
@pytest.fixture(scope="module", params=list(MODELS.keys()))
def model_key(request):
    yield request.param


33
34
35
36
37
38
39
40
41
42
43
44
45
46
@pytest.fixture(scope="module")
def default_server_args():
    return [
        # use half precision for speed and memory savings in CI environment
        "--dtype",
        "bfloat16",
        "--max-model-len",
        "1024",
        "--enforce-eager",
        "--max-num-seqs",
        "128",
    ]


47
48
49
50
51
52
53
54
55
@pytest.fixture(
    scope="module",
    params=[
        "",
        "--enable-chunked-prefill",
        "--disable-frontend-multiprocessing",
        f"--show-hidden-metrics-for-version={PREV_MINOR_VERSION}",
    ],
)
56
def server(model_key, default_server_args, request):
57
58
    if request.param:
        default_server_args.append(request.param)
59

60
61
    model_name = MODELS[model_key]
    with RemoteOpenAIServer(model_name, default_server_args) as remote_server:
62
63
64
65
66
67
68
        yield remote_server


@pytest_asyncio.fixture
async def client(server):
    async with server.get_async_client() as cl:
        yield cl
69
70
71


_PROMPT = "Hello my name is Robert and I love magic"
72
73
74
75
76
77
78
79


def _get_expected_values(num_requests: int, prompt_ids: list[int], max_tokens: int):
    num_prompt_tokens = len(prompt_ids)

    # {metric_family: [(suffix, expected_value)]}
    return {
        "vllm:time_to_first_token_seconds": [("_count", num_requests)],
80
        "vllm:inter_token_latency_seconds": [
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
            ("_count", num_requests * (max_tokens - 1))
        ],
        "vllm:e2e_request_latency_seconds": [("_count", num_requests)],
        "vllm:request_queue_time_seconds": [("_count", num_requests)],
        "vllm:request_inference_time_seconds": [("_count", num_requests)],
        "vllm:request_prefill_time_seconds": [("_count", num_requests)],
        "vllm:request_decode_time_seconds": [("_count", num_requests)],
        "vllm:request_prompt_tokens": [
            ("_sum", num_requests * num_prompt_tokens),
            ("_count", num_requests),
        ],
        "vllm:request_generation_tokens": [
            ("_sum", num_requests * max_tokens),
            ("_count", num_requests),
        ],
        "vllm:request_params_n": [("_count", num_requests)],
        "vllm:request_params_max_tokens": [
            ("_sum", num_requests * max_tokens),
            ("_count", num_requests),
        ],
        "vllm:iteration_tokens_total": [
            (
                "_sum",
                num_requests * (num_prompt_tokens + max_tokens),
            ),
            ("_count", num_requests * max_tokens),
        ],
        "vllm:prompt_tokens": [("_total", num_requests * num_prompt_tokens)],
        "vllm:generation_tokens": [("_total", num_requests * max_tokens)],
        "vllm:request_success": [("_total", num_requests)],
    }
112
113
114


@pytest.mark.asyncio
115
async def test_metrics_counts(
116
117
    server: RemoteOpenAIServer,
    client: openai.AsyncClient,
118
    model_key: str,
119
):
120
121
122
123
124
125
126
127
128
129
    if model_key == "multimodal":
        pytest.skip("Unnecessary test")

    model_name = MODELS[model_key]
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    prompt_ids = tokenizer.encode(_PROMPT)
    num_requests = 10
    max_tokens = 10

    for _ in range(num_requests):
130
131
        # sending a request triggers the metrics to be logged.
        await client.completions.create(
132
133
134
            model=model_name,
            prompt=prompt_ids,
            max_tokens=max_tokens,
135
        )
136

137
    response = requests.get(server.url_for("metrics"))
138
139
140
141
    print(response.text)
    assert response.status_code == HTTPStatus.OK

    # Loop over all expected metric_families
142
143
144
    expected_values = _get_expected_values(num_requests, prompt_ids, max_tokens)
    for metric_family, suffix_values_list in expected_values.items():
        if metric_family not in EXPECTED_METRICS_V1 or (
145
146
147
            not server.show_hidden_metrics
            and metric_family in HIDDEN_DEPRECATED_METRICS
        ):
148
149
            continue

150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
        found_metric = False

        # Check to see if the metric_family is found in the prom endpoint.
        for family in text_string_to_metric_families(response.text):
            if family.name == metric_family:
                found_metric = True

                # Check that each suffix is found in the prom endpoint.
                for suffix, expected_value in suffix_values_list:
                    metric_name_w_suffix = f"{metric_family}{suffix}"
                    found_suffix = False

                    for sample in family.samples:
                        if sample.name == metric_name_w_suffix:
                            found_suffix = True

                            # For each suffix, value sure the value matches
                            # what we expect.
                            assert sample.value == expected_value, (
                                f"{metric_name_w_suffix} expected value of "
                                f"{expected_value} did not match found value "
171
172
                                f"{sample.value}"
                            )
173
174
175
176
177
178
                            break
                    assert found_suffix, (
                        f"Did not find {metric_name_w_suffix} in prom endpoint"
                    )
                break

179
        assert found_metric, f"Did not find {metric_family} in prom endpoint"
180
181


182
183
184
EXPECTED_METRICS_V1 = [
    "vllm:num_requests_running",
    "vllm:num_requests_waiting",
185
186
187
    "vllm:kv_cache_usage_perc",
    "vllm:prefix_cache_queries",
    "vllm:prefix_cache_hits",
188
    "vllm:num_preemptions_total",
189
190
    "vllm:prompt_tokens_total",
    "vllm:generation_tokens_total",
191
    "vllm:iteration_tokens_total",
192
    "vllm:cache_config_info",
193
    "vllm:request_success_total",
194
195
196
197
198
199
    "vllm:request_prompt_tokens_sum",
    "vllm:request_prompt_tokens_bucket",
    "vllm:request_prompt_tokens_count",
    "vllm:request_generation_tokens_sum",
    "vllm:request_generation_tokens_bucket",
    "vllm:request_generation_tokens_count",
200
201
202
203
204
205
    "vllm:request_params_n_sum",
    "vllm:request_params_n_bucket",
    "vllm:request_params_n_count",
    "vllm:request_params_max_tokens_sum",
    "vllm:request_params_max_tokens_bucket",
    "vllm:request_params_max_tokens_count",
206
207
208
209
210
211
    "vllm:time_to_first_token_seconds_sum",
    "vllm:time_to_first_token_seconds_bucket",
    "vllm:time_to_first_token_seconds_count",
    "vllm:inter_token_latency_seconds_sum",
    "vllm:inter_token_latency_seconds_bucket",
    "vllm:inter_token_latency_seconds_count",
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
    "vllm:e2e_request_latency_seconds_sum",
    "vllm:e2e_request_latency_seconds_bucket",
    "vllm:e2e_request_latency_seconds_count",
    "vllm:request_queue_time_seconds_sum",
    "vllm:request_queue_time_seconds_bucket",
    "vllm:request_queue_time_seconds_count",
    "vllm:request_inference_time_seconds_sum",
    "vllm:request_inference_time_seconds_bucket",
    "vllm:request_inference_time_seconds_count",
    "vllm:request_prefill_time_seconds_sum",
    "vllm:request_prefill_time_seconds_bucket",
    "vllm:request_prefill_time_seconds_count",
    "vllm:request_decode_time_seconds_sum",
    "vllm:request_decode_time_seconds_bucket",
    "vllm:request_decode_time_seconds_count",
227
228
]

229
230
231
232
233
EXPECTED_METRICS_MM = [
    "vllm:mm_cache_queries",
    "vllm:mm_cache_hits",
]

234
HIDDEN_DEPRECATED_METRICS: list[str] = [
235
236
237
    "vllm:gpu_cache_usage_perc",
    "vllm:gpu_prefix_cache_queries",
    "vllm:gpu_prefix_cache_hits",
238
]
239

240
241

@pytest.mark.asyncio
242
async def test_metrics_exist(
243
    local_asset_server: LocalAssetServer,
244
245
    server: RemoteOpenAIServer,
    client: openai.AsyncClient,
246
    model_key: str,
247
):
248
249
    model_name = MODELS[model_key]

250
    # sending a request triggers the metrics to be logged.
251
252
253
254
255
256
257
258
    if model_key == "text":
        await client.completions.create(
            model=model_name,
            prompt="Hello, my name is",
            max_tokens=5,
            temperature=0.0,
        )
    else:
259
        # https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
260
261
262
263
264
265
        await client.chat.completions.create(
            model=model_name,
            messages=[
                {
                    "role": "user",
                    "content": [
266
267
268
269
270
271
272
273
                        {
                            "type": "image_url",
                            "image_url": {
                                "url": local_asset_server.url_for(
                                    "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
                                ),
                            },
                        },
274
275
276
277
278
279
280
                        {"type": "text", "text": "What's in this image?"},
                    ],
                }
            ],
            max_tokens=5,
            temperature=0.0,
        )
281

282
    response = requests.get(server.url_for("metrics"))
283
284
    assert response.status_code == HTTPStatus.OK

285
286
287
288
289
290
    expected_metrics = EXPECTED_METRICS_V1
    if model_key == "multimodal":
        # NOTE: Don't use in-place assignment
        expected_metrics = expected_metrics + EXPECTED_METRICS_MM

    for metric in expected_metrics:
291
        if metric in HIDDEN_DEPRECATED_METRICS and not server.show_hidden_metrics:
292
293
            continue
        assert metric in response.text
294
295


296
@pytest.mark.asyncio
297
async def test_abort_metrics_reset(
298
299
    server: RemoteOpenAIServer,
    client: openai.AsyncClient,
300
    model_key: str,
301
):
302
303
304
305
    model_name = MODELS[model_key]
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    prompt_ids = tokenizer.encode(_PROMPT)

306
    running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
307
        server,
308
    )
309
310
311
312
313
314
315
316
317
318
319

    # Expect no running requests or kvcache usage
    assert running_requests == 0
    assert waiting_requests == 0
    assert kv_cache_usage == 0.0

    # Start some long-running requests that we can abort
    tasks = []
    for _ in range(3):
        task = asyncio.create_task(
            client.completions.create(
320
321
                model=model_name,
                prompt=prompt_ids,
322
                max_tokens=100,  # Long generation to give time to abort
323
324
325
                temperature=0.0,
            )
        )
326
327
328
329
330
331
        tasks.append(task)

    # Wait a bit for requests to start processing
    await asyncio.sleep(0.5)

    # Check that we have running requests
332
    running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
333
        server,
334
    )
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352

    # Expect running requests and kvcache usage
    assert running_requests > 0
    assert kv_cache_usage > 0

    # Cancel all tasks to abort the requests
    for task in tasks:
        task.cancel()

    # Wait for cancellations to be processed
    await asyncio.sleep(1.0)

    # Check that metrics have reset to zero
    response = requests.get(server.url_for("metrics"))
    assert response.status_code == HTTPStatus.OK

    # Verify running and waiting requests counts and KV cache usage are zero
    running_requests_after, waiting_requests_after, kv_cache_usage_after = (
353
        _get_running_metrics_from_api(server)
354
    )
355

356
357
358
359
360
361
362
363
364
    assert running_requests_after == 0, (
        f"Expected 0 running requests after abort, got {running_requests_after}"
    )
    assert waiting_requests_after == 0, (
        f"Expected 0 waiting requests after abort, got {waiting_requests_after}"
    )
    assert kv_cache_usage_after == 0, (
        f"Expected 0% KV cache usage after abort, got {kv_cache_usage_after}"
    )
365
366


367
def _get_running_metrics_from_api(server: RemoteOpenAIServer):
368
369
370
371
372
373
374
375
    """Return (running_count, waiting_count, kv_cache_usage)"""

    response = requests.get(server.url_for("metrics"))
    assert response.status_code == HTTPStatus.OK

    # Verify running and waiting requests counts and KV cache usage are zero
    running_requests, waiting_requests, kv_cache_usage = None, None, None

376
    kv_cache_usage_metric = "vllm:kv_cache_usage_perc"
377

378
379
380
381
382
383
384
385
386
387
388
    for family in text_string_to_metric_families(response.text):
        if family.name == "vllm:num_requests_running":
            for sample in family.samples:
                if sample.name == "vllm:num_requests_running":
                    running_requests = sample.value
                    break
        elif family.name == "vllm:num_requests_waiting":
            for sample in family.samples:
                if sample.name == "vllm:num_requests_waiting":
                    waiting_requests = sample.value
                    break
389
        elif family.name == kv_cache_usage_metric:
390
            for sample in family.samples:
391
                if sample.name == kv_cache_usage_metric:
392
393
394
395
396
397
398
399
400
401
                    kv_cache_usage = sample.value
                    break

    assert running_requests is not None
    assert waiting_requests is not None
    assert kv_cache_usage is not None

    return running_requests, waiting_requests, kv_cache_usage


402
def test_metrics_exist_run_batch():
403
    input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "You are a helpful assistant."}}"""  # noqa: E501
404
405
406
407
408

    base_url = "0.0.0.0"
    port = "8001"
    server_url = f"http://{base_url}:{port}"

409
410
411
412
    with (
        tempfile.NamedTemporaryFile("w") as input_file,
        tempfile.NamedTemporaryFile("r") as output_file,
    ):
413
414
        input_file.write(input_batch)
        input_file.flush()
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
        proc = subprocess.Popen(
            [
                sys.executable,
                "-m",
                "vllm.entrypoints.openai.run_batch",
                "-i",
                input_file.name,
                "-o",
                output_file.name,
                "--model",
                "intfloat/multilingual-e5-small",
                "--enable-metrics",
                "--url",
                base_url,
                "--port",
                port,
            ],
        )
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447

        def is_server_up(url):
            try:
                response = requests.get(url)
                return response.status_code == 200
            except requests.ConnectionError:
                return False

        while not is_server_up(server_url):
            time.sleep(1)

        response = requests.get(server_url + "/metrics")
        assert response.status_code == HTTPStatus.OK

        proc.wait()