utils.py 18.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import warnings
5
from collections.abc import Sequence
6
from dataclasses import dataclass
7
from typing import Any
8

9
import torch
10
import torch.nn.functional as F
11
from transformers import PretrainedConfig
12

13
from vllm.config.model import ModelConfig, ModelDType, RunnerOption
14
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
15
16
from vllm.multimodal.processing import InputProcessingContext
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
17

18
from .. import ci_envs
19
20
from .registry import HF_EXAMPLE_MODELS

21
TokensText = tuple[list[int], str]
22
23


24
25
26
27
28
29
30
def check_outputs_equal(
    *,
    outputs_0_lst: Sequence[TokensText],
    outputs_1_lst: Sequence[TokensText],
    name_0: str,
    name_1: str,
):
31
    """
32
    Compare the two sequences generated by different models,
33
34
35
36
    which should be equal.
    """
    assert len(outputs_0_lst) == len(outputs_1_lst)

37
38
39
    for prompt_idx, (outputs_0, outputs_1) in enumerate(
        zip(outputs_0_lst, outputs_1_lst)
    ):
40
41
42
        output_ids_0, output_str_0 = outputs_0
        output_ids_1, output_str_1 = outputs_1

43
        # The text and token outputs should exactly match
44
45
46
47
48
        fail_msg = (
            f"Test{prompt_idx}:"
            f"\n{name_0}:\t{output_str_0!r}"
            f"\n{name_1}:\t{output_str_1!r}"
        )
49
50
51

        assert output_str_0 == output_str_1, fail_msg
        assert output_ids_0 == output_ids_1, fail_msg
52
53


54
55
56
57
58
59
# Representation of generated sequence as a tuple of
# * Token ID list
# * String
# * List of top sample logprobs for each sampled token
#
# Assumes prompt logprobs were not requested.
60
TokensTextLogprobs = tuple[
61
    list[int], str, list[dict[int, float]] | SampleLogprobs | None
62
]
63

64
65
66
67
68
69
70
# Allow for tokens to be represented as str's rather than IDs;
# tuple of
# * Token string representations list
# * String
# * Optional list of top sample logprobs for each sampled token
#
# Assumes prompt logprobs were not requested.
71
TextTextLogprobs = tuple[
72
    list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None
73
]
74

75
76
77
78
79
80
81
# Representation of generated sequence as a tuple of
# * Token ID list
# * String
# * Optional list of top sample logprobs for each sampled token
# * Optional list of top prompt logprobs for each prompt token
#
# Allows prompt logprobs to be requested.
82
TokensTextLogprobsPromptLogprobs = tuple[
83
84
    list[int],
    str,
85
86
    list[dict[int, float]] | SampleLogprobs | None,
    list[dict[int, float] | None] | PromptLogprobs | None,
87
]
88

89

90
91
def check_logprobs_close(
    *,
92
    outputs_0_lst: Sequence[
93
        TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
94
95
    ],
    outputs_1_lst: Sequence[
96
        TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
97
    ],
98
99
    name_0: str,
    name_1: str,
100
    num_outputs_0_skip_tokens: int = 0,
101
    warn_on_mismatch: bool = True,
102
103
104
    always_check_logprobs: bool = False,
) -> None:
    """Compare the logprobs of two sequences generated by different models,
105
    which should be similar but not necessarily equal.
106

107
108
109
110
111
112
113
114
115
116
117
118
    How sample logprobs are compared:
    * `always_check_logprobs == True`: set of highest-logprob token ids
      must match between seq0 and seq1 at all sampled token offsets
    * `always_check_logprobs == False`: highest-logprob token ids are
      only compared at sampled token offsets for which generated token
      ids don't match

    Prompt logprobs must be provided either for both input sequences, or
    for neither. If prompt logprobs are provided, then highest-logprob
    prompt token ids must match between seq0 and seq1 at all prompt token
    offsets.

119
120
121
122
123
124
    Args:
      outputs_0_lst: First sequence to compare
      outputs_0_lst: Second sequence to compare
      name_0: sequence #0 name
      name_1: sequence #1 name
      num_outputs_0_skip_tokens: If > 0, specifies the number of initial
125
126
127
128
129
                                 sequence #0 tokens & logprobs to discard
                                 before comparison, i.e. all
                                 of sequence #1 will be compared to
                                 sequence #0 beginning at index
                                 num_outputs_0_skip_tokens
130
      warn_on_mismatch: Issue a warning if there is token-wise or text-wise
131
                        mismatch between the two sequences
132
      always_check_logprobs: If true, check logprobs even when tokens match
133
    """
134
135
    assert len(outputs_0_lst) == len(outputs_1_lst)

136
    # Loop through responses to each prompt.
137
138
139
    for prompt_idx, (outputs_0, outputs_1) in enumerate(
        zip(outputs_0_lst, outputs_1_lst)
    ):
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
        assert len(outputs_0) == len(outputs_1)
        if len(outputs_0) == 3:
            assert len(outputs_1) == 3
            # Break out tokens, text & sample logprobs
            # (prompt logprobs were not provided)
            output_ids_0, output_str_0, logprobs_0 = outputs_0
            output_ids_1, output_str_1, logprobs_1 = outputs_1
        elif len(outputs_0) == 4:
            assert len(outputs_1) == 4
            # Break out tokens, text, sample logprobs & prompt logprobs
            (
                output_ids_0,
                output_str_0,
                logprobs_0,
                prompt_logprobs_0,
            ) = outputs_0
            (
                output_ids_1,
                output_str_1,
                logprobs_1,
                prompt_logprobs_1,
            ) = outputs_1

            # Test prompt logprobs closeness
164
            if prompt_logprobs_0 is not None and prompt_logprobs_1 is not None:
165
166
167
168
                # Both sequences' prompt logprobs lists are not `None``
                # (although individual list elements may be `None`);
                # for each token's logprobs:
                for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(
169
170
                    zip(prompt_logprobs_0, prompt_logprobs_1)
                ):
171
172
173
                    fail_msg = (
                        f"Prompt logprobs test:"
                        f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}"
174
175
                        f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}"
                    )
176
177
178
179
180
181
182
183
184
185

                    if logprobs_elem_0 is None:
                        # If the seq 0 token's logprobs are `None`,
                        # the seq 1 token's logprobs must be `None`
                        assert logprobs_elem_1 is None, fail_msg
                    else:
                        # If the seq 0 token's logprobs are not `None`,
                        # the seq 1 token's logprobs must not be `None`
                        assert logprobs_elem_1 is not None, fail_msg
                        # Logprobs check: top-k token choices must be the same
186
187
188
                        assert set(logprobs_elem_0.keys()) == set(
                            logprobs_elem_1.keys()
                        ), fail_msg
189
190
            else:
                # Both sequence logprobs lists must be `None`
191
192
193
194
195
                fail_msg = (
                    f"Prompt logprobs test:"
                    f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}"
                    f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}"
                )
196

197
                assert prompt_logprobs_0 is None and prompt_logprobs_1 is None, fail_msg
198
        else:
199
200
201
202
203
            raise ValueError(
                f"Outputs tuple must have 3 or 4 elements but "
                f"{len(outputs_0)} elements were provided: "
                f"{outputs_0}"
            )
204

205
206
207
208
209
        if logprobs_0 is None:
            logprobs_0 = [None] * len(output_ids_0)
        if logprobs_1 is None:
            logprobs_1 = [None] * len(output_ids_1)

210
211
212
213
214
215
216
217
218
        # Skip specified number of initial sequence #0 tokens
        # & logprobs, leaving output text as-is for simplicity
        # (text mismatches may generate warnings but do not
        # cause the test to fail.)
        if num_outputs_0_skip_tokens < 0:
            raise ValueError("num_outputs_0_skip_tokens must be non-negative")
        output_ids_0 = output_ids_0[num_outputs_0_skip_tokens:]
        logprobs_0 = logprobs_0[num_outputs_0_skip_tokens:]

219
        # Loop through generated tokens.
220
221
222
        for idx, (output_id_0, output_id_1) in enumerate(
            zip(output_ids_0, output_ids_1)
        ):
223
224
225
226
227
228
            is_tok_mismatch = output_id_0 != output_id_1

            # If generated tokens don't match
            # or it is desired to always check logprobs,
            # then
            if is_tok_mismatch or always_check_logprobs:
229
230
231
                logprobs_elem_0 = logprobs_0[idx]
                logprobs_elem_1 = logprobs_1[idx]

232
                # Each predicted token must be in top N logprobs of the other
233
                fail_msg = (
234
                    f"Test{prompt_idx}:"
235
                    f"\nMatched tokens:\t{output_ids_0[:idx]}"
236
                    f"\n{name_0}:\t{output_str_0!r}\t{logprobs_elem_0}"
237
238
                    f"\n{name_1}:\t{output_str_1!r}\t{logprobs_elem_1}"
                )
239
240
241
242
243
244

                assert logprobs_elem_0 is not None, fail_msg
                assert logprobs_elem_1 is not None, fail_msg
                assert output_id_0 in logprobs_elem_1, fail_msg
                assert output_id_1 in logprobs_elem_0, fail_msg

245
                if warn_on_mismatch and is_tok_mismatch:
246
247
248
249
250
251
                    with warnings.catch_warnings():
                        # This ensures that repeated warnings are shown
                        # in the output, not just the first occurrence
                        warnings.simplefilter("always")

                        warnings.warn(fail_msg, stacklevel=2)
252
253
254

                # Break out since sequences will now diverge.
                break
255
256
257
258
        else:
            if output_str_0 != output_str_1 and warn_on_mismatch:
                # The token outputs exactly match,
                # so the text outputs should exactly match as well
259
260
261
262
263
                fail_msg = (
                    f"Test{prompt_idx}:"
                    f"\n{name_0}:\t{output_str_0!r}"
                    f"\n{name_1}:\t{output_str_1!r}"
                )
264
265
266
267
268
269
270

                with warnings.catch_warnings():
                    # This ensures that repeated warnings are shown
                    # in the output, not just the first occurrence
                    warnings.simplefilter("always")

                    warnings.warn(fail_msg, stacklevel=2)
271
272


273
def build_model_context(
274
    model_id: str,
275
    runner: RunnerOption = "auto",
276
    dtype: ModelDType = "auto",
277
278
279
    model_config_kwargs: dict[str, Any] | None = None,
    mm_processor_kwargs: dict[str, Any] | None = None,
    limit_mm_per_prompt: dict[str, int] | None = None,
280
    mm_processor_cache_gb: int = 0,
281
):
282
    """Creates an InputProcessingContext for a given model.
283

284
    Args:
285
        model_id: ID of the model being considered.
286
287
288
289
290
        mm_processor_kwargs: optional processor kwargs for to be leveraged
            in the input processor, mapper, dummy data creation, etc.
        limit_mm_per_prompt: Multimodal limits.

    Returns:
291
        InputProcessingContext for the model being considered.
292
    """
293
294
295
296
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
    model_info.check_available_online(on_fail="skip")
    model_info.check_transformers_version(on_fail="skip")

297
    model_config_kwargs = model_config_kwargs or {}
298
    limit_mm_per_prompt = limit_mm_per_prompt or {}
299
    model_config = ModelConfig(
300
        model_id,
301
        runner=runner,
302
303
        tokenizer=model_info.tokenizer or model_id,
        tokenizer_mode=model_info.tokenizer_mode,
304
        revision=model_info.revision,
305
        trust_remote_code=model_info.trust_remote_code,
306
        dtype=dtype,
307
308
309
        seed=0,
        mm_processor_kwargs=mm_processor_kwargs,
        limit_mm_per_prompt=limit_mm_per_prompt,
310
        mm_processor_cache_gb=mm_processor_cache_gb,
311
        hf_overrides=model_info.hf_overrides,
312
313
        skip_tokenizer_init=model_info.skip_tokenizer_init,
        enforce_eager=model_info.enforce_eager,
314
        **model_config_kwargs,
315
    )
316
317
318
319
320

    return InputProcessingContext(
        model_config,
        tokenizer=cached_tokenizer_from_config(model_config),
    )
321
322
323
324
325
326
327
328
329
330
331
332
333


def check_embeddings_close(
    *,
    embeddings_0_lst: Sequence[list[float]],
    embeddings_1_lst: Sequence[list[float]],
    name_0: str,
    name_1: str,
    tol: float = 1e-3,
) -> None:
    assert len(embeddings_0_lst) == len(embeddings_1_lst)

    for prompt_idx, (embeddings_0, embeddings_1) in enumerate(
334
335
        zip(embeddings_0_lst, embeddings_1_lst)
    ):
336
        assert len(embeddings_0) == len(embeddings_1), (
337
338
            f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}"
        )
339

340
341
342
        sim = F.cosine_similarity(
            torch.tensor(embeddings_0), torch.tensor(embeddings_1), dim=0
        )
343

344
345
346
347
348
349
        fail_msg = (
            f"Test{prompt_idx}:"
            f"\nCosine similarity: \t{sim:.4f}"
            f"\n{name_0}:\t{embeddings_0[:16]!r}"
            f"\n{name_1}:\t{embeddings_1[:16]!r}"
        )
350
351
352
353
354
355
356
357
358
359
360

        assert sim >= 1 - tol, fail_msg


def matryoshka_fy(tensor: torch.Tensor, dimensions: int):
    tensor = torch.tensor(tensor)
    tensor = tensor[..., :dimensions]
    tensor = F.normalize(tensor, p=2, dim=1)
    return tensor


361
362
363
364
365
366
367
def softmax(data):
    if data.shape[-1] == 1:
        return F.sigmoid(data)
    else:
        return F.softmax(data, dim=-1)


368
369
@dataclass
class ModelInfo:
370
371
    name: str
    architecture: str = ""
372
    dtype: str = "auto"
373
    max_model_len: int | None = None
374
    hf_dtype: str = "float32"
375
    hf_overrides: dict[str, Any] | None = None
376
    default_pooling_type: str = ""
377
    enable_test: bool = True
378
379


380
381
@dataclass
class EmbedModelInfo(ModelInfo):
382
    mteb_score: float | None = None
383
    is_matryoshka: bool = False
384
    matryoshka_dimensions: list[int] | None = None
385
386
387


@dataclass
388
389
390
391
class CLSPoolingEmbedModelInfo(EmbedModelInfo):
    default_pooling_type: str = "CLS"


392
@dataclass
393
394
395
396
class LASTPoolingEmbedModelInfo(EmbedModelInfo):
    default_pooling_type: str = "LAST"


397
398
@dataclass
class RerankModelInfo(ModelInfo):
399
    mteb_score: float | None = None
400
401


402
@dataclass
403
404
405
406
class CLSPoolingRerankModelInfo(RerankModelInfo):
    default_pooling_type: str = "CLS"


407
@dataclass
408
409
410
411
class LASTPoolingRerankModelInfo(RerankModelInfo):
    default_pooling_type: str = "LAST"


412
413
414
@dataclass
class GenerateModelInfo(ModelInfo):
    hf_dtype: str = "auto"
415
    hf_ppl: float | None = None
416
417


418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
def get_vllm_extra_kwargs(model_info: ModelInfo, vllm_extra_kwargs):
    # A model family has many models with the same architecture,
    # and we don't need to test each one.
    if not ci_envs.VLLM_CI_NO_SKIP and not model_info.enable_test:
        import pytest

        pytest.skip("Skipping test.")

    # Allow vllm to test using the given dtype, such as float32
    vllm_extra_kwargs = vllm_extra_kwargs or {}
    vllm_extra_kwargs["dtype"] = ci_envs.VLLM_CI_DTYPE or model_info.dtype

    # Allow vllm to test using hf_overrides
    if model_info.hf_overrides is not None:
        vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides

    # Allow changing the head dtype used by vllm in tests
    if ci_envs.VLLM_CI_HEAD_DTYPE is not None:
        if "hf_overrides" not in vllm_extra_kwargs:
            vllm_extra_kwargs["hf_overrides"] = {}
        vllm_extra_kwargs["hf_overrides"]["head_dtype"] = ci_envs.VLLM_CI_HEAD_DTYPE

    # Allow control over whether tests use enforce_eager
    if ci_envs.VLLM_CI_ENFORCE_EAGER is not None:
        vllm_extra_kwargs["enforce_eager"] = ci_envs.VLLM_CI_ENFORCE_EAGER

    return vllm_extra_kwargs


447
448
def dummy_hf_overrides(
    hf_config: PretrainedConfig,
449
450
    *,
    model_arch: str = "",
451
    exist_overrides: dict[str, Any] | None = None,
452
    use_original_num_layers: bool = False,
453
454
455
456
457
458
459
460
461
462
463
) -> PretrainedConfig:
    """
    Dummy HF overrides function used to create dummy model
    with only minimum nums of layer.
    """
    hf_config.update(exist_overrides or {})

    text_config = hf_config.get_text_config()

    # Ensure at least 2 expert per group
    # Since `grouped_topk` assumes top-2
464
    n_group = getattr(text_config, "n_group", None)
465
466
467
468
    num_experts = n_group * 2 if n_group is not None else 2

    # we use three layers for Gemma-3n to check
    # both normal layer and kv_shared_layer
469
470
    if use_original_num_layers:
        # Use the original number of layers from the config
471
472
        num_layers = getattr(text_config, "num_layers", 1)
        num_hidden_layers = getattr(text_config, "num_hidden_layers", 1)
473
474
475
    else:
        # Use minimal layers for testing
        num_layers = 1
476
        num_hidden_layers = 3 if model_arch == "Gemma3nForConditionalGeneration" else 1
477

XuruiYang's avatar
XuruiYang committed
478
    update_dict = {
479
        "num_layers": num_layers,
480
481
        # For Gemma-3n
        "num_kv_shared_layers": 1,
XuruiYang's avatar
XuruiYang committed
482
483
    }

484
485
486
487
488
489
    class DummyConfig:
        hf_text_config = text_config

    # Only set MoE related config when the model has MoE layers.
    # Otherwise all models detected as MoE by _get_transformers_backend_cls.
    if ModelConfig.get_num_experts(DummyConfig) > 0:
490
491
492
493
494
495
496
497
498
499
500
        update_dict.update(
            {
                "num_experts": num_experts,
                "num_experts_per_tok": 2,
                "num_local_experts": num_experts,
                # Otherwise there will not be any expert layers
                "first_k_dense_replace": 0,
                # To avoid OOM on DeepSeek-V3
                "n_routed_experts": num_experts,
            }
        )
501

XuruiYang's avatar
XuruiYang committed
502
    # Update num_hidden_layers for non-Longcat architectures
503
    if model_arch != "LongcatFlashForCausalLM" and model_arch != "LongCatFlashMTPModel":
XuruiYang's avatar
XuruiYang committed
504
505
506
        update_dict["num_hidden_layers"] = num_hidden_layers

    text_config.update(update_dict)
507
508

    if hasattr(hf_config, "vision_config"):
509
510
511
512
513
514
        hf_config.vision_config.update(
            {
                "num_layers": 1,
                "num_hidden_layers": 1,
            }
        )
515
516
517

    # e.g.: ibm-granite/granite-speech-3.3-2b
    if hasattr(hf_config, "encoder_config"):
518
519
520
521
522
523
        hf_config.encoder_config.update(
            {
                "num_layers": 1,
                "num_hidden_layers": 1,
            }
        )
524
525
526

    # e.g.: Qwen/Qwen2-Audio-7B-Instruct
    if hasattr(hf_config, "audio_config"):
527
528
529
530
531
532
533
        hf_config.audio_config.update(
            {
                "num_layers": 1,
                "num_hidden_layers": 1,
                "encoder_layers": 1,
            }
        )
534
535

    return hf_config
536
537


538
539
def check_transformers_version(
    model: str,
540
541
    min_transformers_version: str | None = None,
    max_transformers_version: str | None = None,
542
):
543
544
    from .registry import _HfExamplesInfo

545
546
547
548
549
    return _HfExamplesInfo(
        model,
        min_transformers_version=min_transformers_version,
        max_transformers_version=max_transformers_version,
    ).check_transformers_version(on_fail="skip")