test_regression.py 2.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
7
8
"""Containing tests that check for regressions in vLLM's behavior.

It should include tests that are reported by users and making sure they
will never happen again.

"""
9

10
11
import gc

12
import pytest
13
14
import torch

15
from vllm import LLM, SamplingParams
16
from vllm.platforms import current_platform
17
18


19
@pytest.mark.skip(reason="In V1, we reject tokens > max_seq_len")
20
21
22
def test_duplicated_ignored_sequence_group():
    """https://github.com/vllm-project/vllm/issues/1655"""

23
24
25
26
27
28
    sampling_params = SamplingParams(temperature=0.01, top_p=0.1, max_tokens=256)
    llm = LLM(
        model="distilbert/distilgpt2",
        max_num_batched_tokens=4096,
        tensor_parallel_size=1,
    )
29
30
31
32
33
34
    prompts = ["This is a short prompt", "This is a very long prompt " * 1000]
    outputs = llm.generate(prompts, sampling_params=sampling_params)

    assert len(prompts) == len(outputs)


35
def test_max_tokens_none():
36
37
38
39
40
41
    sampling_params = SamplingParams(temperature=0.01, top_p=0.1, max_tokens=None)
    llm = LLM(
        model="distilbert/distilgpt2",
        max_num_batched_tokens=4096,
        tensor_parallel_size=1,
    )
42
43
44
45
46
47
    prompts = ["Just say hello!"]
    outputs = llm.generate(prompts, sampling_params=sampling_params)

    assert len(prompts) == len(outputs)


48
def test_gc():
49
    llm = LLM(model="distilbert/distilgpt2", enforce_eager=True)
50
51
52
    del llm

    gc.collect()
53
    torch.accelerator.empty_cache()
54
55
56
57
58
59
60
61

    # The memory allocated for model and KV cache should be released.
    # The memory allocated for PyTorch and others should be less than 50MB.
    # Usually, it's around 10MB.
    allocated = torch.cuda.memory_allocated()
    assert allocated < 50 * 1024 * 1024


62
def test_model_from_modelscope(monkeypatch: pytest.MonkeyPatch):
63
    # model: https://modelscope.cn/models/qwen/Qwen1.5-0.5B-Chat/summary
64
65
    with monkeypatch.context() as m:
        m.setenv("VLLM_USE_MODELSCOPE", "True")
66
67
68
        # Don't use HF_TOKEN for ModelScope repos, otherwise it will fail
        # with 400 Client Error: Bad Request.
        m.setenv("HF_TOKEN", "")
69
70
        attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
        llm = LLM(model="qwen/Qwen1.5-0.5B-Chat", attention_backend=attn_backend)
71
72
73
74
75
76
77
78
79
80
81

        prompts = [
            "Hello, my name is",
            "The president of the United States is",
            "The capital of France is",
            "The future of AI is",
        ]
        sampling_params = SamplingParams(temperature=0.8, top_p=0.95)

        outputs = llm.generate(prompts, sampling_params)
        assert len(outputs) == 4