conftest.py 54.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
import contextlib
import pathlib
from copy import deepcopy
6
7
8

from tblib import pickling_support

9
10
# ruff: noqa

11
12
13
14
15
# Install support for pickling exceptions so that we can nicely propagate
# failures from tests running in a subprocess.
# This should be run before any custom exception subclasses are defined.
pickling_support.install()

16
import http.server
17
import json
18
import math
19
import mimetypes
20
import os
21
import socket
22
import tempfile
23
24
import threading
from collections.abc import Generator
25
from contextlib import nullcontext
26
from enum import Enum
27
from typing import Any, Callable, TypedDict, TypeVar, cast, TYPE_CHECKING, Optional
Woosuk Kwon's avatar
Woosuk Kwon committed
28

29
import numpy as np
Woosuk Kwon's avatar
Woosuk Kwon committed
30
31
import pytest
import torch
32
import torch.nn as nn
33
import torch.nn.functional as F
34
from huggingface_hub import snapshot_download
35
from PIL import Image
36
37
38
39
40
41
42
from transformers import (
    AutoConfig,
    AutoModelForCausalLM,
    AutoTokenizer,
    BatchEncoding,
    BatchFeature,
)
43
from transformers.models.auto.auto_factory import _BaseAutoModelClass
Woosuk Kwon's avatar
Woosuk Kwon committed
44

45
46
47
48
49
from tests.models.utils import (
    TokensTextLogprobs,
    TokensTextLogprobsPromptLogprobs,
    softmax,
)
50
from vllm import LLM, SamplingParams, envs
51
from vllm.assets.audio import AudioAsset
52
from vllm.assets.image import ImageAsset
53
from vllm.assets.video import VideoAsset
54
from vllm.config.model import ConvertOption, RunnerOption, _get_and_verify_dtype
55
from vllm.connections import global_http_connection
56
57
58
59
60
from vllm.distributed import (
    cleanup_dist_env_and_memory,
    init_distributed_environment,
    initialize_model_parallel,
)
61
from vllm.logger import init_logger
62
from vllm.logprobs import Logprob
63
from vllm.multimodal.media import MediaWithBytes
64
from vllm.multimodal.utils import fetch_image
65
from vllm.outputs import RequestOutput
66
from vllm.sampling_params import BeamSearchParams
67
from vllm.transformers_utils.utils import maybe_model_redirect
68
from vllm.utils.collection_utils import is_list_of
69
from vllm.utils.torch_utils import set_default_torch_num_threads
70

71
72
73
from torch._inductor.utils import fresh_cache


74
75
76
77
78
if TYPE_CHECKING:
    from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
    from transformers.generation.utils import GenerateOutput


79
logger = init_logger(__name__)
Woosuk Kwon's avatar
Woosuk Kwon committed
80

81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

@pytest.fixture
def sample_json_schema():
    return {
        "type": "object",
        "properties": {
            "name": {"type": "string"},
            "age": {"type": "integer"},
            "skills": {
                "type": "array",
                "items": {
                    "type": "string",
                },
            },
            "grade": {
                "type": "string",
                "pattern": "^[A-D]$",
            },
            "email": {
                "type": "string",
                "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
            },
            "work_history": {
                "type": "array",
                "items": {
                    "type": "object",
                    "properties": {
                        "company": {"type": "string"},
                        "duration": {
                            "type": "number",
                            "minimum": 0.0,
                            "maximum": 100.0,
                        },
                        "position": {"type": "string"},
                    },
                    "required": ["company", "duration", "position"],
                    "additionalProperties": False,
                },
                "minItems": 0,
                "maxItems": 3,
            },
        },
        "required": ["name", "age", "skills", "grade", "email", "work_history"],
        "additionalProperties": False,
        "minProperties": 1,
        "maxProperties": 10,
    }


130
131
132
_TEST_DIR = os.path.dirname(__file__)
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
133
_SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")
134

Cyrus Leung's avatar
Cyrus Leung committed
135
_M = TypeVar("_M")
136

137
_PromptMultiModalInput = list[_M] | list[list[_M]]
Cyrus Leung's avatar
Cyrus Leung committed
138
139

PromptImageInput = _PromptMultiModalInput[Image.Image]
140
PromptAudioInput = _PromptMultiModalInput[tuple[np.ndarray, int]]
Cyrus Leung's avatar
Cyrus Leung committed
141
PromptVideoInput = _PromptMultiModalInput[np.ndarray]
142

143

144
def _read_prompts(filename: str) -> list[str]:
145
    with open(filename) as f:
146
147
        prompts = f.readlines()
        return prompts
Woosuk Kwon's avatar
Woosuk Kwon committed
148
149


150
class ImageAssetPrompts(TypedDict):
151
152
    stop_sign: str
    cherry_blossom: str
153
154


155
class ImageTestAssets(list[ImageAsset]):
156
    def __init__(self) -> None:
157
158
159
160
161
162
        super().__init__(
            [
                ImageAsset("stop_sign"),
                ImageAsset("cherry_blossom"),
            ]
        )
163

164
    def prompts(self, prompts: ImageAssetPrompts) -> list[str]:
165
166
167
168
169
170
        """
        Convenience method to define the prompt for each test image.

        The order of the returned prompts matches the order of the
        assets when iterating through this object.
        """
171
        return [prompts["stop_sign"], prompts["cherry_blossom"]]
172
173


174
175
class VideoAssetPrompts(TypedDict):
    baby_reading: str
176
177


178
class VideoTestAssets(list[VideoAsset]):
179
    def __init__(self) -> None:
180
181
182
183
184
        super().__init__(
            [
                VideoAsset("baby_reading"),
            ]
        )
185

186
187
    def prompts(self, prompts: VideoAssetPrompts) -> list[str]:
        return [prompts["baby_reading"]]
188
189


190
class AudioAssetPrompts(TypedDict):
191
192
193
194
    mary_had_lamb: str
    winning_call: str


195
class AudioTestAssets(list[AudioAsset]):
196
    def __init__(self) -> None:
197
198
199
200
201
202
        super().__init__(
            [
                AudioAsset("mary_had_lamb"),
                AudioAsset("winning_call"),
            ]
        )
203

204
    def prompts(self, prompts: AudioAssetPrompts) -> list[str]:
205
206
        return [prompts["mary_had_lamb"], prompts["winning_call"]]

207

208
IMAGE_ASSETS = ImageTestAssets()
209
"""Singleton instance of {class}`ImageTestAssets`."""
210
VIDEO_ASSETS = VideoTestAssets()
211
"""Singleton instance of {class}`VideoTestAssets`."""
212
AUDIO_ASSETS = AudioTestAssets()
213
"""Singleton instance of {class}`AudioTestAssets`."""
214
215


216
217
218
219
220
221
222
@pytest.fixture(autouse=True)
def init_test_http_connection():
    # pytest_asyncio may use a different event loop per test
    # so we need to make sure the async client is created anew
    global_http_connection.reuse_client = False


223
224
@pytest.fixture
def dist_init():
225
226
    from tests.utils import ensure_current_vllm_config

227
    temp_file = tempfile.mkstemp()[1]
228
229
230
231
232
233
234
235
236
237
238

    with ensure_current_vllm_config():
        init_distributed_environment(
            world_size=1,
            rank=0,
            distributed_init_method=f"file://{temp_file}",
            local_rank=0,
            backend="nccl",
        )
        initialize_model_parallel(1, 1)
        yield
239
    cleanup_dist_env_and_memory()
240
241


242
243
244
245
246
247
248
@pytest.fixture
def default_vllm_config():
    """Set a default VllmConfig for tests that directly test CustomOps or pathways
    that use get_current_vllm_config() outside of a full engine context.
    """
    from vllm.config import VllmConfig, set_current_vllm_config

249
250
251
    config = VllmConfig()
    with set_current_vllm_config(config):
        yield config
252
253


254
@pytest.fixture()
255
def should_do_global_cleanup_after_test(request) -> bool:
256
257
258
259
    """Allow subdirectories to skip global cleanup by overriding this fixture.
    This can provide a ~10x speedup for non-GPU unit tests since they don't need
    to initialize torch.
    """
260

261
    return not request.node.get_closest_marker("skip_global_cleanup")
262
263


264
@pytest.fixture(autouse=True)
265
def cleanup_fixture(should_do_global_cleanup_after_test: bool):
266
    yield
267
    if should_do_global_cleanup_after_test:
268
        cleanup_dist_env_and_memory()
269
270


271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
@pytest.fixture
def workspace_init():
    """Initialize the workspace manager for tests that need it.

    This fixture initializes the workspace manager with a CUDA device
    if available, and resets it after the test completes. Tests that
    create a full vLLM engine should NOT use this fixture as the engine
    will initialize the workspace manager itself.
    """
    from vllm.v1.worker.workspace import (
        init_workspace_manager,
        reset_workspace_manager,
    )

    if torch.cuda.is_available():
        device = torch.device("cuda:0")
        init_workspace_manager(device)
    yield
    reset_workspace_manager()


292
293
294
295
296
297
@pytest.fixture(autouse=True)
def dynamo_reset():
    yield
    torch._dynamo.reset()


Woosuk Kwon's avatar
Woosuk Kwon committed
298
@pytest.fixture
299
def example_prompts() -> list[str]:
300
    return [prompt for filename in _TEST_PROMPTS for prompt in _read_prompts(filename)]
301
302


303
304
305
306
307
308
@pytest.fixture
def example_system_message() -> str:
    with open(_SYS_MSG) as f:
        return f.read()


309
310
class DecoderPromptType(Enum):
    """For encoder/decoder models only."""
311

312
313
314
315
316
    CUSTOM = 1
    NONE = 2
    EMPTY_STR = 3


317
@pytest.fixture
318
def example_long_prompts() -> list[str]:
319
    return [prompt for filename in _LONG_PROMPTS for prompt in _read_prompts(filename)]
Woosuk Kwon's avatar
Woosuk Kwon committed
320
321


322
@pytest.fixture(scope="session")
323
def image_assets() -> ImageTestAssets:
324
325
326
    return IMAGE_ASSETS


327
@pytest.fixture(scope="session")
328
def video_assets() -> VideoTestAssets:
329
330
331
    return VIDEO_ASSETS


332
@pytest.fixture(scope="session")
333
def audio_assets() -> AudioTestAssets:
334
335
336
    return AUDIO_ASSETS


337
_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
338
_R = TypeVar("_R")
339

Woosuk Kwon's avatar
Woosuk Kwon committed
340
341

class HfRunner:
342
    def get_default_device(self):
343
        from vllm.platforms import current_platform
344

345
        return "cpu" if current_platform.is_cpu() else current_platform.device_type
346

347
    def wrap_device(self, x: _T, device: str | None = None) -> _T:
348
        if x is None or isinstance(x, (bool,)):
349
350
            return x

351
        if device is None:
352
            device = self.device
353

354
355
        if isinstance(x, dict):
            return {k: self.wrap_device(v, device) for k, v in x.items()}
356

357
358
359
360
        if hasattr(x, "device") and x.device.type == device:
            return x

        return x.to(device)
361

Woosuk Kwon's avatar
Woosuk Kwon committed
362
363
364
    def __init__(
        self,
        model_name: str,
365
        dtype: str = "auto",
366
        *,
367
        model_kwargs: dict[str, Any] | None = None,
368
        trust_remote_code: bool = True,
369
        is_sentence_transformer: bool = False,
370
        is_cross_encoder: bool = False,
371
        skip_tokenizer_init: bool = False,
372
        auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
373
        # Set this to avoid hanging issue
374
        default_torch_num_threads: int | None = None,
375
    ) -> None:
376
377
378
379
380
        init_ctx = (
            nullcontext()
            if default_torch_num_threads is None
            else set_default_torch_num_threads(default_torch_num_threads)
        )
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398

        with init_ctx:
            self._init(
                model_name=model_name,
                dtype=dtype,
                model_kwargs=model_kwargs,
                trust_remote_code=trust_remote_code,
                is_sentence_transformer=is_sentence_transformer,
                is_cross_encoder=is_cross_encoder,
                skip_tokenizer_init=skip_tokenizer_init,
                auto_cls=auto_cls,
            )

    def _init(
        self,
        model_name: str,
        dtype: str = "auto",
        *,
399
        model_kwargs: dict[str, Any] | None = None,
400
401
402
403
404
        trust_remote_code: bool = True,
        is_sentence_transformer: bool = False,
        is_cross_encoder: bool = False,
        skip_tokenizer_init: bool = False,
        auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
Woosuk Kwon's avatar
Woosuk Kwon committed
405
    ) -> None:
406
        model_name = maybe_model_redirect(model_name)
407
        self.model_name = model_name
408

409
410
        self.config = AutoConfig.from_pretrained(
            model_name,
411
            trust_remote_code=trust_remote_code,
412
        )
413
414
415
416
417
418
419
420
421
        # HF runner should use the HF config so that it's consistent with the HF model
        if self.config.__module__.startswith("vllm.transformers_utils.configs"):
            from transformers.models.auto.configuration_auto import CONFIG_MAPPING

            del CONFIG_MAPPING._extra_content[self.config.model_type]
            self.config = AutoConfig.from_pretrained(
                model_name,
                trust_remote_code=trust_remote_code,
            )
422
        self.device = self.get_default_device()
423
        self.dtype = dtype = _get_and_verify_dtype(
424
425
426
427
            self.model_name,
            self.config,
            dtype=dtype,
            is_pooling_model=is_sentence_transformer or is_cross_encoder,
428
            config_format="hf",
429
        )
430
431

        model_kwargs = model_kwargs if model_kwargs is not None else {}
432
        model_kwargs.setdefault("dtype", dtype)
433

434
        if is_sentence_transformer:
435
436
            # Lazy init required for AMD CI
            from sentence_transformers import SentenceTransformer
437
438
439
440
441

            self.model = SentenceTransformer(
                model_name,
                device=self.device,
                model_kwargs=model_kwargs,
442
                trust_remote_code=trust_remote_code,
443
            )
444
445
446
        elif is_cross_encoder:
            # Lazy init required for AMD CI
            from sentence_transformers import CrossEncoder
447
448
449
450
451

            self.model = CrossEncoder(
                model_name,
                device=self.device,
                automodel_args=model_kwargs,
452
                trust_remote_code=trust_remote_code,
453
            )
454
        else:
455
456
457
458
459
460
461
            model = cast(
                nn.Module,
                auto_cls.from_pretrained(
                    model_name,
                    trust_remote_code=trust_remote_code,
                    **model_kwargs,
                ),
462
463
            )

464
            # in case some unquantized custom models are not in same dtype
465
466
467
            if getattr(model, "quantization_method", None) is None and any(
                p.dtype != self.dtype for p in model.parameters()
            ):
468
469
                model = model.to(dtype=self.dtype)

470
471
472
473
            if (
                getattr(model, "quantization_method", None) != "bitsandbytes"
                and len({p.device for p in model.parameters()}) < 2
            ):
474
                model = model.to(device=self.device)
475
476

            self.model = model
477

478
        if not skip_tokenizer_init:
479
480
481
482
483
            self.tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast" = (
                AutoTokenizer.from_pretrained(
                    model_name,
                    trust_remote_code=trust_remote_code,
                )
484
            )
485

486
        # don't put this import at the top level
487
        # it will call torch.accelerator.device_count()
488
        from transformers import AutoProcessor
489

490
491
        self.processor = AutoProcessor.from_pretrained(
            model_name,
492
            trust_remote_code=trust_remote_code,
493
        )
494
495
        if skip_tokenizer_init:
            self.tokenizer = self.processor.tokenizer
Woosuk Kwon's avatar
Woosuk Kwon committed
496

497
    def get_inputs(
Woosuk Kwon's avatar
Woosuk Kwon committed
498
        self,
499
500
501
502
        prompts: list[str] | list[list[int]],
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
503
        tokenization_kwargs: dict[str, Any] | None = None,
504
    ) -> list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]]:
505
        if images is not None:
506
            assert len(prompts) == len(images)
507

508
509
510
511
512
513
        if videos is not None:
            assert len(prompts) == len(videos)

        if audios is not None:
            assert len(prompts) == len(audios)

514
        all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = []
515
        for i, prompt in enumerate(prompts):
516
            if isinstance(prompt, str):
517
518
519
520
521
522
523
524
525
526
527
528
                # Create a copy to avoid modifying the original dict
                processor_kwargs = (
                    tokenization_kwargs.copy()
                    if tokenization_kwargs is not None
                    else {}
                )
                processor_kwargs.update(
                    {
                        "text": prompt,
                        "return_tensors": "pt",
                    }
                )
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
                if images is not None and (image := images[i]) is not None:
                    processor_kwargs["images"] = image
                if videos is not None and (video := videos[i]) is not None:
                    processor_kwargs["videos"] = video
                if audios is not None and (audio_inputs := audios[i]) is not None:
                    # HACK - not all processors take sampling_rate; we should
                    # clean this up in the future.
                    if len(audio_inputs) == 2:
                        audio, sr = audio_inputs
                        processor_kwargs["audio"] = audio
                        processor_kwargs["sampling_rate"] = sr
                    else:
                        processor_kwargs["audio"] = audio_inputs

                inputs = self.processor(**processor_kwargs)
                if isinstance(inputs, BatchFeature):
                    inputs = inputs.to(dtype=self.dtype)
                all_inputs.append(inputs)
            else:
                # check that prompt is (batched) list of integers (token ids)
                if not is_list_of(prompt, typ=int, check="all"):
                    raise ValueError(
                        "Prompt must be a list of ints corresponding to the prompt token ids."
                    )
                # check that no multimodal input is provided
                if images or videos or audios:
                    raise ValueError(
                        "When providing prompt token ids multimodal inputs are not supported."
                    )
                input_dict = {
                    "input_ids": torch.tensor(prompt, dtype=torch.long).unsqueeze(0),
                }
                all_inputs.append(input_dict)
562
563
564

        return all_inputs

565
566
567
568
569
570
571
572
573
    def get_prompt_embeddings(self, prompts: list[str]) -> list[torch.Tensor]:
        all_inputs = self.get_inputs(prompts)
        embeddings = []
        for inputs in all_inputs:
            input_ids = self.wrap_device(inputs)["input_ids"]
            embedding = self.model.get_input_embeddings()(input_ids).squeeze(0)
            embeddings.append(embedding)
        return embeddings

574
    def classify(self, prompts: list[str]) -> list[list[float]]:
575
576
        # output is final logits
        all_inputs = self.get_inputs(prompts)
577
        outputs: list[list[float]] = []
578
579
        problem_type = getattr(self.config, "problem_type", "")

580
581
        for inputs in all_inputs:
            output = self.model(**self.wrap_device(inputs))
582
583
584

            assert isinstance(output.logits, torch.Tensor)

585
586
587
588
589
            if problem_type == "regression":
                logits = output.logits[0].tolist()
            elif problem_type == "multi_label_classification":
                logits = output.logits.sigmoid()[0].tolist()
            else:
590
                logits = softmax(output.logits)[0].tolist()
591
592
593
594
            outputs.append(logits)

        return outputs

595
596
    def generate(
        self,
597
598
599
600
        prompts: list[str] | list[list[int]],
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
601
        **kwargs: Any,
602
    ) -> list[tuple[list[list[int]], list[str]]]:
603
604
605
        all_inputs = self.get_inputs(
            prompts, images=images, videos=videos, audios=audios
        )
606

607
        outputs: list[tuple[list[list[int]], list[str]]] = []
608
        for inputs in all_inputs:
609
            output_ids: torch.Tensor = self.model.generate(
610
                **self.wrap_device(inputs),
Woosuk Kwon's avatar
Woosuk Kwon committed
611
612
613
                use_cache=True,
                **kwargs,
            )
614
            output_str = self.processor.batch_decode(
Woosuk Kwon's avatar
Woosuk Kwon committed
615
616
617
                output_ids,
                skip_special_tokens=True,
                clean_up_tokenization_spaces=False,
618
            )
619
            outputs.append((output_ids.cpu().tolist(), output_str))
Woosuk Kwon's avatar
Woosuk Kwon committed
620
621
622
623
        return outputs

    def generate_greedy(
        self,
624
        prompts: list[str] | list[list[int]],
Woosuk Kwon's avatar
Woosuk Kwon committed
625
        max_tokens: int,
626
627
628
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
629
        **kwargs: Any,
630
    ) -> list[tuple[list[int], str]]:
631
632
633
634
635
636
637
638
639
        outputs = self.generate(
            prompts,
            do_sample=False,
            max_new_tokens=max_tokens,
            images=images,
            videos=videos,
            audios=audios,
            **kwargs,
        )
640

641
        return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]
642
643
644

    def generate_beam_search(
        self,
645
        prompts: list[str],
646
647
        beam_width: int,
        max_tokens: int,
648
649
650
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
651
    ) -> list[tuple[list[list[int]], list[str]]]:
652
653
654
655
656
657
658
659
660
661
        outputs = self.generate(
            prompts,
            do_sample=False,
            max_new_tokens=max_tokens,
            num_beams=beam_width,
            num_return_sequences=beam_width,
            images=images,
            videos=videos,
            audios=audios,
        )
662

663
664
665
666
        for i in range(len(outputs)):
            output_ids, output_str = outputs[i]
            for j in range(len(output_ids)):
                output_ids[j] = [
667
                    x for x in output_ids[j] if x != self.tokenizer.pad_token_id
668
669
670
                ]
            outputs[i] = (output_ids, output_str)
        return outputs
Woosuk Kwon's avatar
Woosuk Kwon committed
671

672
673
    def generate_greedy_logprobs(
        self,
674
        prompts: list[str],
675
        max_tokens: int,
676
677
678
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
679
        **kwargs: Any,
680
    ) -> list[list[torch.Tensor]]:
681
682
683
        all_inputs = self.get_inputs(
            prompts, images=images, videos=videos, audios=audios
        )
684

685
        all_logprobs: list[list[torch.Tensor]] = []
686
        for inputs in all_inputs:
687
            output: "GenerateOutput" = self.model.generate(
688
                **self.wrap_device(inputs),
689
690
691
692
693
                use_cache=True,
                do_sample=False,
                max_new_tokens=max_tokens,
                output_hidden_states=True,
                return_dict_in_generate=True,
694
                **kwargs,
695
            )
696
            seq_logprobs = self._hidden_states_to_seq_logprobs(output.hidden_states)
697
698
699
            all_logprobs.append(seq_logprobs)
        return all_logprobs

700
    def _hidden_states_to_seq_logprobs(
701
        self,
702
703
        hidden_states: tuple[tuple[torch.Tensor, ...], ...],
    ) -> list[torch.Tensor]:
704
705
        output_embeddings = self.model.get_output_embeddings()

706
        seq_logprobs: list[torch.Tensor] = []
707
708
709
        for _, hidden_state in enumerate(hidden_states):
            last_hidden_states = hidden_state[-1][0]
            logits = torch.matmul(
710
711
712
713
                last_hidden_states.to(
                    device=output_embeddings.weight.device,
                    dtype=output_embeddings.weight.dtype,
                ),
714
                output_embeddings.weight.t(),
715
            )
716
717
            if getattr(output_embeddings, "bias", None) is not None:
                logits += output_embeddings.bias.unsqueeze(0)
718
719
720
            logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
            seq_logprobs.append(logprobs)

721
722
723
724
        return seq_logprobs

    def _hidden_states_to_logprobs(
        self,
725
        hidden_states: tuple[tuple[torch.Tensor, ...], ...],
726
        num_logprobs: int | None,
727
    ) -> tuple[list[dict[int, float]], int]:
728
729
730
        seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)
        output_len = len(hidden_states)

731
        # convert to dict
732
        seq_logprobs_lst: list[dict[int, float]] = []
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
        for tok_idx, tok_logprobs in enumerate(seq_logprobs):
            # drop prompt logprobs
            if tok_idx == 0:
                tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
            topk = tok_logprobs.topk(num_logprobs)

            tok_logprobs_dct = {}
            for token_id, logprob in zip(topk.indices[0], topk.values[0]):
                tok_logprobs_dct[token_id.item()] = logprob.item()

            seq_logprobs_lst.append(tok_logprobs_dct)

        return (
            seq_logprobs_lst,
            output_len,
        )

750
751
    def generate_greedy_logprobs_limit(
        self,
752
        prompts: list[str],
753
        max_tokens: int,
754
755
756
757
        num_logprobs: int | None,
        images: PromptImageInput | None = None,
        audios: PromptAudioInput | None = None,
        videos: PromptVideoInput | None = None,
758
        use_cache: bool = True,
759
        **kwargs: Any,
760
    ) -> list[TokensTextLogprobs]:
761
762
763
        all_inputs = self.get_inputs(
            prompts, images=images, videos=videos, audios=audios
        )
764

765
766
767
        all_logprobs: list[list[dict[int, float]]] = []
        all_output_ids: list[list[int]] = []
        all_output_strs: list[str] = []
768

769
        for inputs in all_inputs:
770
            output: "GenerateOutput" = self.model.generate(
771
                **self.wrap_device(inputs),
772
                use_cache=use_cache,
773
774
775
776
                do_sample=False,
                max_new_tokens=max_tokens,
                output_hidden_states=True,
                return_dict_in_generate=True,
777
                **kwargs,
778
779
            )

780
781
782
783
784
785
            # Encoder-decoder models return decoder_hidden_states instead of
            # hidden_states
            hidden_states = (
                getattr(output, "hidden_states", None) or output.decoder_hidden_states
            )

786
787
788
            (
                seq_logprobs_lst,
                output_len,
789
            ) = self._hidden_states_to_logprobs(hidden_states, num_logprobs)
790
791
792
793
794
795
796

            all_logprobs.append(seq_logprobs_lst)
            seq_ids = output.sequences[0]
            output_len = len(seq_logprobs_lst)
            output_ids = seq_ids[-output_len:]
            all_output_ids.append(output_ids.tolist())
            all_output_strs.append(self.tokenizer.decode(output_ids))
797

798
        outputs = zip(all_output_ids, all_output_strs, all_logprobs)
799
800
801
802
        return [
            (output_ids, output_str, output_logprobs)
            for output_ids, output_str, output_logprobs in outputs
        ]
803

804
    def encode(self, prompts: list[str], *args, **kwargs) -> list[list[torch.Tensor]]:
805
        return self.model.encode(prompts, *args, **kwargs)
806

807
808
    def predict(self, prompts: list[list[str]], *args, **kwargs) -> torch.Tensor:
        return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs)
809

810
811
812
813
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
814
        del self.model
815
        cleanup_dist_env_and_memory()
816

Woosuk Kwon's avatar
Woosuk Kwon committed
817

Cyrus Leung's avatar
Cyrus Leung committed
818
@pytest.fixture(scope="session")
Woosuk Kwon's avatar
Woosuk Kwon committed
819
820
821
822
823
def hf_runner():
    return HfRunner


class VllmRunner:
824
825
    """
    The default value of some arguments have been modified from
826
    {class}`~vllm.LLM` as follows:
827

828
829
830
    - `trust_remote_code`: Set to `True` instead of `False` for convenience.
    - `seed`: Set to `0` instead of `None` for test reproducibility.
    - `max_model_len`: Set to `1024` instead of `None` to reduce memory usage.
831
832
    - `block_size`: To reduce memory usage, set default to `64` if on XPU
        devices, otherwise default to `16`.
833
834
    - `enable_chunked_prefill`: Set to `False` instead of `None` for
      test reproducibility.
835
    - `enforce_eager`: Set to `False` to test CUDA graph.
836
    """
Woosuk Kwon's avatar
Woosuk Kwon committed
837
838
839
840

    def __init__(
        self,
        model_name: str,
841
842
        runner: RunnerOption = "auto",
        convert: ConvertOption = "auto",
843
        tokenizer_name: str | None = None,
844
        tokenizer_mode: str = "auto",
845
        trust_remote_code: bool = True,
846
        seed: int = 0,
847
        max_model_len: int | None = 1024,
848
        dtype: str = "auto",
849
        disable_log_stats: bool = True,
850
        tensor_parallel_size: int = 1,
851
        block_size: int = 16 if not torch.xpu.is_available() else 64,
852
853
        enable_chunked_prefill: bool | None = False,
        enforce_eager: bool | None = False,
854
        # Set this to avoid hanging issue
855
        default_torch_num_threads: int | None = None,
856
        **kwargs,
Woosuk Kwon's avatar
Woosuk Kwon committed
857
    ) -> None:
858
859
860
861
862
        init_ctx = (
            nullcontext()
            if default_torch_num_threads is None
            else set_default_torch_num_threads(default_torch_num_threads)
        )
863

864
        if not kwargs.get("compilation_config", None):
865
866
867
868
            # Note(@tdoublep): This is set to 4 because some tests (e.g., hybrid
            # model tests) may set max_num_seqs=4. If min cudagraph_capture_size is
            # set to larger than max_num_seqs, then it will lead to *no* graphs
            # being captured which can trigger edge cases that we don't handle yet.
869
            kwargs["compilation_config"] = {"cudagraph_capture_sizes": [4]}
870

871
872
873
874
875
876
877
878
            # Make sure we have atleast one cudagraph large enough for a single decode.
            if (speculative_config := kwargs.get("speculative_config")) and (
                num_speculative_tokens := speculative_config["num_speculative_tokens"]
            ):
                kwargs["compilation_config"]["cudagraph_capture_sizes"].append(
                    num_speculative_tokens + 1
                )

879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
        with init_ctx:
            self.llm = LLM(
                model=model_name,
                runner=runner,
                convert=convert,
                tokenizer=tokenizer_name,
                tokenizer_mode=tokenizer_mode,
                trust_remote_code=trust_remote_code,
                dtype=dtype,
                seed=seed,
                enforce_eager=enforce_eager,
                disable_log_stats=disable_log_stats,
                tensor_parallel_size=tensor_parallel_size,
                max_model_len=max_model_len,
                block_size=block_size,
                enable_chunked_prefill=enable_chunked_prefill,
                **kwargs,
            )
Woosuk Kwon's avatar
Woosuk Kwon committed
897

898
    def get_inputs(
Woosuk Kwon's avatar
Woosuk Kwon committed
899
        self,
900
901
902
903
        prompts: list[str]
        | list[torch.Tensor]
        | list[list[int]]
        | list[dict[str, Any]],
904
905
906
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
907
    ) -> list[dict[str, Any]]:
908
909
910
        if any(
            x is not None and len(x) != len(prompts) for x in [images, videos, audios]
        ):
911
            raise ValueError(
912
913
                "All non-None multimodal inputs must have the same length as prompts"
            )
914

915
        inputs = list[dict[str, Any]]()
916
        for i, prompt in enumerate(prompts):
917
918
919
920
921
            # If we're passing an encoder/decoder prompt, we assume it
            # already contains the multimodal data in the prompt
            if isinstance(prompt, dict):
                assert images is None and audios is None and videos is None
                inputs.append(prompt.copy())
922
            else:
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
                prompt_dict = dict[str, Any]()
                if isinstance(prompt, str):
                    prompt_dict["prompt"] = prompt
                elif isinstance(prompt, list):
                    prompt_dict["prompt_token_ids"] = prompt
                else:
                    prompt_dict["prompt_embeds"] = prompt

                multi_modal_data = dict[str, Any]()
                if images is not None and (image := images[i]) is not None:
                    multi_modal_data["image"] = image
                if videos is not None and (video := videos[i]) is not None:
                    multi_modal_data["video"] = video
                if audios is not None and (audio := audios[i]) is not None:
                    multi_modal_data["audio"] = audio
938

939
940
                if multi_modal_data:
                    prompt_dict["multi_modal_data"] = multi_modal_data
941

942
                inputs.append(prompt_dict)
943
944
945
946
947

        return inputs

    def generate(
        self,
948
        prompts: list[str] | list[torch.Tensor] | list[list[int]],
949
        sampling_params: SamplingParams,
950
951
952
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
953
        return_logprobs: bool = False,
954
        **kwargs: Any,
955
    ) -> list[tuple[list[list[int]], list[str]]] | tuple[list, list]:
956
        inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
957

958
959
960
        req_outputs = self.llm.generate(
            inputs, sampling_params=sampling_params, **kwargs
        )
961

962
        outputs: list[tuple[list[list[int]], list[str]]] = []
963
        logprobs = []
Woosuk Kwon's avatar
Woosuk Kwon committed
964
965
966
        for req_output in req_outputs:
            prompt_str = req_output.prompt
            prompt_ids = req_output.prompt_token_ids
967
968
            req_sample_output_ids: list[list[int]] = []
            req_sample_output_strs: list[str] = []
969
            req_logprobs = []
970
971
            for sample in req_output.outputs:
                output_str = sample.text
972
                output_ids = list(sample.token_ids)
973
                req_sample_output_ids.append(prompt_ids + output_ids)
974
                req_sample_output_strs.append((prompt_str or "") + output_str)
975
976
                if sample.logprobs:
                    req_logprobs.extend(sample.logprobs)
977
            outputs.append((req_sample_output_ids, req_sample_output_strs))
978
979
            logprobs.append(req_logprobs)
        return outputs if not return_logprobs else (outputs, logprobs)
Woosuk Kwon's avatar
Woosuk Kwon committed
980

981
    @staticmethod
982
    def _final_steps_generate_w_logprobs(
983
        req_outputs: list[RequestOutput],
984
        include_prompt_token_ids: bool = False,
985
986
    ) -> list[TokensTextLogprobsPromptLogprobs]:
        outputs: list[TokensTextLogprobsPromptLogprobs] = []
987
        for req_output in req_outputs:
988
            assert len(req_output.outputs) > 0
989
990
            for sample in req_output.outputs:
                output_str = sample.text
991
                output_ids = list(sample.token_ids)
992
                output_logprobs = sample.logprobs
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
            if include_prompt_token_ids:
                outputs.append(
                    (  # type: ignore[arg-type]
                        output_ids,
                        output_str,
                        output_logprobs,
                        req_output.prompt_token_ids,
                        req_output.prompt_logprobs,
                    )
                )
            else:
                outputs.append(
                    (
                        output_ids,
                        output_str,
                        output_logprobs,
                        req_output.prompt_logprobs,
                    )
                )

1013
1014
        return outputs

1015
1016
    def generate_w_logprobs(
        self,
1017
        prompts: list[str],
1018
        sampling_params: SamplingParams,
1019
1020
1021
        images: PromptImageInput | None = None,
        audios: PromptAudioInput | None = None,
        videos: PromptVideoInput | None = None,
1022
        include_prompt_token_ids: bool = False,
1023
        **kwargs: Any,
1024
    ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:
1025
1026
1027
1028
1029
1030
1031
        inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)

        req_outputs = self.llm.generate(
            inputs, sampling_params=sampling_params, **kwargs
        )

        toks_str_logsprobs_prompt_logprobs = self._final_steps_generate_w_logprobs(
1032
            req_outputs, include_prompt_token_ids
1033
        )
1034
        # Omit prompt logprobs if not required by sampling params
1035
1036
1037
1038
1039
        return (
            [x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]
            if sampling_params.prompt_logprobs is None
            else toks_str_logsprobs_prompt_logprobs
        )
1040

Woosuk Kwon's avatar
Woosuk Kwon committed
1041
1042
    def generate_greedy(
        self,
1043
        prompts: list[str] | list[torch.Tensor] | list[list[int]],
Woosuk Kwon's avatar
Woosuk Kwon committed
1044
        max_tokens: int,
1045
1046
1047
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
1048
        **kwargs: Any,
1049
    ) -> list[tuple[list[int], str]]:
Woosuk Kwon's avatar
Woosuk Kwon committed
1050
        greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
1051
1052
1053
1054
1055
1056
1057
1058
1059
        outputs = self.generate(
            prompts,
            greedy_params,
            images=images,
            videos=videos,
            audios=audios,
            **kwargs,
        )
        return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]
1060

1061
1062
    def generate_greedy_logprobs(
        self,
1063
        prompts: list[str],
1064
        max_tokens: int,
1065
1066
1067
1068
1069
1070
1071
        num_logprobs: int | None,
        num_prompt_logprobs: int | None = None,
        images: PromptImageInput | None = None,
        audios: PromptAudioInput | None = None,
        videos: PromptVideoInput | None = None,
        stop_token_ids: list[int] | None = None,
        stop: list[str] | None = None,
1072
        **kwargs: Any,
1073
    ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:
1074
1075
1076
1077
        greedy_logprobs_params = SamplingParams(
            temperature=0.0,
            max_tokens=max_tokens,
            logprobs=num_logprobs,
1078
            prompt_logprobs=num_prompt_logprobs,
1079
            stop_token_ids=stop_token_ids,
1080
1081
            stop=stop,
        )
1082

1083
1084
1085
1086
1087
1088
1089
1090
        return self.generate_w_logprobs(
            prompts,
            greedy_logprobs_params,
            images=images,
            audios=audios,
            videos=videos,
            **kwargs,
        )
1091

1092
1093
1094
    def generate_prompt_perplexity(
        self, prompts: list[str], mask: Optional[list[str]] = None
    ) -> list[float]:
1095
1096
1097
1098
1099
1100
        """
        Return the perplexity score associated with generating the prompts

        :param prompts: list of prompts to score
        :return: perplexity score of each prompt
        """
1101
1102
1103
        outputs = self.generate_greedy_logprobs(
            prompts, max_tokens=1, num_logprobs=None, num_prompt_logprobs=0
        )
1104

1105
1106
1107
1108
1109
1110
        mask_prefix_lens = (
            [len(self.llm.get_tokenizer()(prefix)["input_ids"]) for prefix in mask]
            if mask is not None
            else [0 for _ in range(len(prompts))]
        )

1111
        perplexities = []
1112
        for output, mask_prefix_len in zip(outputs, mask_prefix_lens):
1113
            output = cast(TokensTextLogprobsPromptLogprobs, output)
1114
            token_datas = cast(list[dict[int, Logprob] | None], output[3])
1115
            assert token_datas[0] is None
1116

1117
            token_log_probs = []
1118
            for token_data in token_datas[mask_prefix_len + 1 :]:
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
                assert token_data is not None
                assert len(token_data) == 1
                token_log_prob = list(token_data.values())[0].logprob
                token_log_probs.append(token_log_prob)

            perplexity = math.exp(-sum(token_log_probs) / len(token_log_probs))
            perplexities.append(perplexity)

        return perplexities

1129
    def generate_beam_search(
1130
        self,
1131
        prompts: list[str],
1132
1133
        beam_width: int,
        max_tokens: int,
1134
1135
1136
1137
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
        concurrency_limit: int | None = None,
1138
    ) -> list[tuple[list[list[int]], list[str]]]:
1139
1140
1141
1142
1143
1144
1145
        inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)

        outputs = self.llm.beam_search(
            inputs,
            BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens),
            concurrency_limit=concurrency_limit,
        )
1146
1147
1148
1149
1150
1151
1152
        returned_outputs = []
        for output in outputs:
            token_ids = [x.tokens for x in output.sequences]
            texts = [x.text for x in output.sequences]
            returned_outputs.append((token_ids, texts))
        return returned_outputs

1153
    def classify(self, prompts: list[str]) -> list[list[float]]:
1154
        req_outputs = self.llm.classify(prompts)
1155
1156
        return [req_output.outputs.probs for req_output in req_outputs]

1157
1158
1159
    def embed(
        self,
        prompts: list[str],
1160
1161
1162
        images: PromptImageInput | None = None,
        videos: PromptVideoInput | None = None,
        audios: PromptAudioInput | None = None,
1163
1164
1165
1166
        *args,
        **kwargs,
    ) -> list[list[float]]:
        inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
Cyrus Leung's avatar
Cyrus Leung committed
1167

1168
        req_outputs = self.llm.embed(inputs, *args, **kwargs)
Cyrus Leung's avatar
Cyrus Leung committed
1169
        return [req_output.outputs.embedding for req_output in req_outputs]
1170

1171
1172
1173
1174
1175
1176
    def token_embed(self, prompts: list[str]) -> list[list[float]]:
        req_outputs = self.llm.encode(prompts, pooling_task="token_embed")
        return [req_output.outputs.data for req_output in req_outputs]

    def token_classify(self, prompts: list[str]) -> list[list[float]]:
        req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
1177
1178
        return [req_output.outputs.data for req_output in req_outputs]

1179
1180
1181
1182
    def reward(self, prompts: list[str]) -> list[list[float]]:
        req_outputs = self.llm.reward(prompts)
        return [req_output.outputs.data for req_output in req_outputs]

1183
1184
    def score(
        self,
1185
1186
        text_1: list[str] | str,
        text_2: list[str] | str,
1187
1188
        *args,
        **kwargs,
1189
    ) -> list[float]:
1190
        req_outputs = self.llm.score(text_1, text_2, *args, **kwargs)
1191
        return [req_output.outputs.score for req_output in req_outputs]
1192

1193
    def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
1194
        return self.llm.apply_model(func)
1195

1196
1197
1198
    def get_llm(self) -> LLM:
        return self.llm

1199
1200
1201
    def collective_rpc(self, *args, **kwargs):
        return self.llm.collective_rpc(*args, **kwargs)

1202
1203
1204
1205
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
1206
1207
1208
1209
1210
1211
1212
1213
1214
        # Explicitly shutdown the engine core to release GPU resources
        # This is needed because when executing consecutive tests, the GC
        # might not be fast enough in shutting down the llm engine. This can lead to OOMs
        # because when the next test starts some GPU memory is still in use.
        try:
            self.llm.llm_engine.engine_core.shutdown()
        except Exception:
            # Ignore shutdown errors as cleanup will still proceed
            pass
1215
        del self.llm
1216
        cleanup_dist_env_and_memory()
1217

Woosuk Kwon's avatar
Woosuk Kwon committed
1218

1219
@pytest.fixture(scope="session")
Woosuk Kwon's avatar
Woosuk Kwon committed
1220
1221
def vllm_runner():
    return VllmRunner
1222
1223


1224
1225
1226
@pytest.fixture()
def temporary_enable_log_propagate():
    import logging
1227

1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
    logger = logging.getLogger("vllm")
    logger.propagate = True
    yield
    logger.propagate = False


@pytest.fixture()
def caplog_vllm(temporary_enable_log_propagate, caplog):
    # To capture vllm log, we should enable propagate=True temporarily
    # because caplog depends on logs propagated to the root logger.
    yield caplog
1239
1240


1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
@pytest.fixture()
def caplog_mp_fork():
    """
    This fixture enables capturing logs from a forked MP subprocess.
    It should be used in conjunction with caplog_vllm.

    By default, subprocess logs do not go through the parent process.
    We instead create a queue listener in the parent process which
    forwards logs to the logger's other handlers, and add a QueueHandler
    to the root logger. Forked subprocesses will inherit the root logger
    and pass their messages to the queue, which the listener will forward
    to the root logger, which can be captured by caplog.

    Note that this workaround only works for fork; with spawn, the subprocess
    reinitializes logging and does not automatically inherit the queue.
    We'd have to manually pass the queue to the subprocess at the spawn point.
    See caplog_mp_spawn below.
    """

    @contextlib.contextmanager
    def ctx():
        import logging.handlers
        import multiprocessing as mp

        logger_queue: mp.Queue[logging.LogRecord] = mp.Queue()
        logger = logging.getLogger()
        handlers = logger.handlers

        # The listener works on a background thread, not inherited by the child.
        queue_listener = logging.handlers.QueueListener(logger_queue, *handlers)
        queue_listener.start()

        # Add queue handler after creating the listener to avoid cycle
        logger.addHandler(logging.handlers.QueueHandler(logger_queue))
        yield
        queue_listener.stop()

    return ctx


class LogHolder:
    def __init__(self):
        self.text = None


@pytest.fixture()
def caplog_mp_spawn(tmp_path, monkeypatch):
    """
    This fixture enables capturing logs from a forked MP subprocess.
    It does not require caplog_vllm (but it only contains logs from the child).

    By default, subprocess logs do not go through the parent process.
    We instead add a FileHandler to the config so the spawned child process
    writes its logs to a temp file.
    In the parent, we read the file and return the contents.

    Note: this method could be extended to fork by either reconfiguring logging
    in the parent or using a SocketHandler:
    https://docs.python.org/3/howto/logging-cookbook.html#sending-and-receiving-logging-events-across-a-network # noqa: E501
    """

    @contextlib.contextmanager
    def ctx(level: int | str):
        from vllm.logger import DEFAULT_LOGGING_CONFIG

        config_path = tmp_path / "vllm_logging_config.json"
        log_path = tmp_path / "vllm.log"
        log_holder = LogHolder()

        config = deepcopy(DEFAULT_LOGGING_CONFIG)
        if envs.VLLM_LOGGING_CONFIG_PATH:
            path = pathlib.Path(envs.VLLM_LOGGING_CONFIG_PATH)
            assert path.exists()
            config = json.loads(path.read_text())

        config["loggers"]["vllm"]["handlers"] += ["vllm_file"]
        config["handlers"]["vllm_file"] = {
            "class": "logging.FileHandler",
            "formatter": "vllm",
            "level": level,
            "filename": log_path.as_posix(),
        }
1323
        config["loggers"]["vllm"]["level"] = level
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336

        config_path.write_text(json.dumps(config))

        with monkeypatch.context() as monkeypatch_ctx:
            monkeypatch_ctx.setenv("VLLM_LOGGING_CONFIG_PATH", config_path.as_posix())
            monkeypatch_ctx.setenv("VLLM_CONFIGURE_LOGGING", "1")
            yield log_holder

        log_holder.text = log_path.read_text()

    return ctx


1337
1338
1339
1340
1341
@pytest.fixture(scope="session")
def num_gpus_available():
    """Get number of GPUs without initializing the CUDA context
    in current process."""

1342
    from vllm.platforms import current_platform
1343

1344
    return current_platform.device_count()
1345
1346
1347


temp_dir = tempfile.gettempdir()
1348
1349
_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
1350
_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
1351
1352
1353
1354


@pytest.fixture
def dummy_opt_path():
1355
1356
    json_path = os.path.join(_dummy_opt_path, "config.json")
    if not os.path.exists(_dummy_opt_path):
1357
1358
1359
1360
1361
        snapshot_download(
            repo_id="facebook/opt-125m",
            local_dir=_dummy_opt_path,
            ignore_patterns=["*.bin", "*.bin.index.json", "*.pt", "*.h5", "*.msgpack"],
        )
1362
        assert os.path.exists(json_path)
1363
        with open(json_path) as f:
1364
1365
1366
1367
            config = json.load(f)
        config["architectures"] = ["MyOPTForCausalLM"]
        with open(json_path, "w") as f:
            json.dump(config, f)
1368
1369
1370
1371
1372
1373
1374
    return _dummy_opt_path


@pytest.fixture
def dummy_llava_path():
    json_path = os.path.join(_dummy_llava_path, "config.json")
    if not os.path.exists(_dummy_llava_path):
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
        snapshot_download(
            repo_id="llava-hf/llava-1.5-7b-hf",
            local_dir=_dummy_llava_path,
            ignore_patterns=[
                "*.bin",
                "*.bin.index.json",
                "*.pt",
                "*.h5",
                "*.msgpack",
                "*.safetensors",
            ],
        )
1387
        assert os.path.exists(json_path)
1388
        with open(json_path) as f:
1389
1390
1391
1392
1393
            config = json.load(f)
        config["architectures"] = ["MyLlava"]
        with open(json_path, "w") as f:
            json.dump(config, f)
    return _dummy_llava_path
1394
1395
1396
1397
1398
1399


@pytest.fixture
def dummy_gemma2_embedding_path():
    json_path = os.path.join(_dummy_gemma2_embedding_path, "config.json")
    if not os.path.exists(_dummy_gemma2_embedding_path):
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
        snapshot_download(
            repo_id="BAAI/bge-multilingual-gemma2",
            local_dir=_dummy_gemma2_embedding_path,
            ignore_patterns=[
                "*.bin",
                "*.bin.index.json",
                "*.pt",
                "*.h5",
                "*.msgpack",
                "*.safetensors",
            ],
        )
1412
        assert os.path.exists(json_path)
1413
        with open(json_path) as f:
1414
1415
1416
1417
1418
            config = json.load(f)
        config["architectures"] = ["MyGemma2Embedding"]
        with open(json_path, "w") as f:
            json.dump(config, f)
    return _dummy_gemma2_embedding_path
1419
1420
1421
1422
1423


# Add the flag `--optional` to allow run tests
# that are marked with @pytest.mark.optional
def pytest_addoption(parser):
1424
1425
1426
    parser.addoption(
        "--optional", action="store_true", default=False, help="run optional test"
    )
1427
1428
1429
1430
1431
1432
1433
1434
1435


def pytest_collection_modifyitems(config, items):
    if config.getoption("--optional"):
        # --optional given in cli: do not skip optional tests
        return
    skip_optional = pytest.mark.skip(reason="need --optional option to run")
    for item in items:
        if "optional" in item.keywords:
1436
            item.add_marker(skip_optional)
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448


@pytest.fixture(scope="session")
def cli_config_file():
    """Return the path to the CLI config file."""
    return os.path.join(_TEST_DIR, "config", "test_config.yaml")


@pytest.fixture(scope="session")
def cli_config_file_with_model():
    """Return the path to the CLI config file with model."""
    return os.path.join(_TEST_DIR, "config", "test_config_with_model.yaml")
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495


class AssetHandler(http.server.BaseHTTPRequestHandler):
    # _IMAGE_CACHE : Dict[str, bytes] = {}

    def log_message(self, *args, **kwargs):
        pass

    def do_GET(self):
        # Accepts paths like: /1280px-Venn_diagram_rgb.jpg
        filename = self.path.lstrip("/")
        if not filename or "." not in filename:
            self.send_error(404, "Missing filename (expected /<name>.<ext>)")
            return

        base, ext = filename.rsplit(".", 1)
        ext = ext.lower()

        if ext not in ["jpg", "png"]:
            self.send_error(404, f"Unsupported extension: .{ext}")
            return

        try:
            data = ImageAsset(base).read_bytes(ext=ext)
        except Exception as e:
            self.send_error(500, f"Failed to load asset: {ext} {base} {e} ")
            return

        ctype, _ = mimetypes.guess_type(filename)
        if ctype is None:
            ctype = {"jpg": "image/jpg", "png": "image/png"}[ext]
        self.send_response(200)
        self.send_header("Content-Type", ctype)
        self.send_header("Content-Length", str(len(data)))
        self.end_headers()
        self.wfile.write(data)


def _find_free_port() -> int:
    with socket.socket() as s:
        s.bind(("127.0.0.1", 0))
        return s.getsockname()[1]


class LocalAssetServer:
    address: str
    port: int
1496
1497
    server: http.server.ThreadingHTTPServer | None
    thread: threading.Thread | None
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507

    def __init__(self, address: str = "127.0.0.1") -> None:
        self.address = address
        self.port = -1
        self.server = None
        self.thread = None

    def __enter__(self):
        self.port = _find_free_port()
        self.server = http.server.ThreadingHTTPServer(
1508
1509
1510
            (self.address, self.port), AssetHandler
        )
        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
        self.thread.start()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.server:
            self.server.shutdown()
            del self.server

        if self.thread:
            self.thread.join()
            del self.thread

        if exc_type is None:
            return None

        return False

    @property
    def base_url(self) -> str:
        assert self.port is not None
        return f"http://{self.address}:{self.port}"

    def url_for(self, name: str) -> str:
        """e.g., name='RGBA_comp.png' -> 'http://127.0.0.1:PORT/RGBA_comp.png'"""
        return f"{self.base_url}/{name}"

    def get_image_asset(self, name: str) -> Image.Image:
1538
1539
1540
1541
1542
        image = fetch_image(self.url_for(name))
        # Unwrap MediaWithBytes if present
        if isinstance(image, MediaWithBytes):
            image = image.media
        return image
1543
1544
1545
1546
1547


@pytest.fixture(scope="session")
def local_asset_server() -> Generator[LocalAssetServer, None, None]:
    """
1548
    Starts a thread based HTTP server bound to 127.0.0.1 on a random free port.
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
    The server currently servers images at:
    http://127.0.0.1:<port>/<name>.<ext>
    """
    with LocalAssetServer() as srv:
        yield srv


@pytest.fixture
def image_url(request, local_asset_server) -> str:
    # request.param is one of the IMAGE_ASSETS filenames
    name = request.param
    return local_asset_server.url_for(name)


@pytest.fixture
def image_urls(request, local_asset_server) -> list[str]:
    """Indirect fixture: takes a list of names, returns list of full URLs."""
    names: list[str] = request.param
    return [local_asset_server.url_for(name) for name in names]
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580


@pytest.fixture
def disable_deepgemm_ue8m0(monkeypatch):
    from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used

    with monkeypatch.context() as monkeypatch_ctx:
        monkeypatch_ctx.setenv("VLLM_USE_DEEP_GEMM_E8M0", "0")
        is_deep_gemm_e8m0_used.cache_clear()
        yield
        # Clear cache so the next time it is used it is processed with the
        # default VLLM_USE_DEEP_GEMM_E8M0  setting.
        is_deep_gemm_e8m0_used.cache_clear()
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593


@pytest.fixture(autouse=True)
def clean_gpu_memory_between_tests():
    if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1":
        yield
        return

    # Wait for GPU memory to be cleared before starting the test
    import gc

    from tests.utils import wait_for_gpu_memory_to_clear

1594
    num_gpus = torch.accelerator.device_count()
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
    if num_gpus > 0:
        try:
            wait_for_gpu_memory_to_clear(
                devices=list(range(num_gpus)),
                threshold_ratio=0.1,
            )
        except ValueError as e:
            logger.info("Failed to clean GPU memory: %s", e)

    yield

    # Clean up GPU memory after the test
    if torch.cuda.is_available():
1608
        torch.accelerator.empty_cache()
1609
        gc.collect()
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620


@pytest.fixture
def use_fresh_inductor_cache():
    """
    Use a fresh inductor cache for the test.
    This is useful to ensure that the test is not affected by the
    previous test calls.
    """
    with fresh_cache():
        yield
1621
1622


1623
1624
1625
1626
1627
1628
1629
1630
@pytest.fixture
def fresh_vllm_cache(monkeypatch, use_fresh_inductor_cache):
    """Temporary VLLM_CACHE_ROOT combined with a fresh inductor cache."""
    with tempfile.TemporaryDirectory() as tmp_dir:
        monkeypatch.setenv("VLLM_CACHE_ROOT", tmp_dir)
        yield tmp_dir


1631
1632
1633
1634
@pytest.fixture(scope="function")
def enable_pickle(monkeypatch):
    """`LLM.apply_model` requires pickling a function."""
    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657


@pytest.fixture(scope="function")
def disable_log_dedup(monkeypatch):
    """
    Disable log deduplication such that warning_once and info_once always print.
    """

    # Patch logger._print_warning_once to remove the lru_cache decorator
    from vllm import logger

    original_print_warning_once = logger._print_warning_once
    original_print_info_once = logger._print_info_once
    original_print_debug_once = logger._print_debug_once

    logger._print_warning_once = original_print_warning_once.__wrapped__
    logger._print_info_once = original_print_info_once.__wrapped__
    logger._print_debug_once = original_print_debug_once.__wrapped__

    yield
    logger._print_warning_once = original_print_warning_once
    logger._print_info_once = original_print_info_once
    logger._print_debug_once = original_print_debug_once