test_online.py 9.75 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5

import pytest
import requests
6
7
import torch
import torch.nn.functional as F
8

9
from tests.utils import RemoteOpenAIServer
10
11
from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
DTYPE = "float32"  # Use float32 to avoid NaN issue


@pytest.fixture(scope="module")
def server():
    args = [
        "--enforce-eager",
        "--max-model-len",
        "512",
        "--dtype",
        DTYPE,
    ]

    with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
        yield remote_server


@pytest.mark.parametrize("model_name", [MODEL_NAME])
32
def test_single_input_classification(server: RemoteOpenAIServer, model_name: str):
33
34
35
36
    input_text = "This product was excellent and exceeded my expectations"

    classification_response = requests.post(
        server.url_for("classify"),
37
        json={"model": model_name, "input": input_text},
38
39
40
    )

    classification_response.raise_for_status()
41
    output = ClassificationResponse.model_validate(classification_response.json())
42
43
44
45
46
47
48
49

    assert output.object == "list"
    assert output.model == MODEL_NAME
    assert len(output.data) == 1
    assert hasattr(output.data[0], "label")
    assert hasattr(output.data[0], "probs")


50
51
52
53
54
55
56
57
58
59
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_add_special_tokens_false(server: RemoteOpenAIServer, model_name: str):
    response = requests.post(
        server.url_for("classify"),
        json={"model": model_name, "input": "hello", "add_special_tokens": False},
    )
    response.raise_for_status()
    ClassificationResponse.model_validate(response.json())


60
@pytest.mark.parametrize("model_name", [MODEL_NAME])
61
def test_multiple_inputs_classification(server: RemoteOpenAIServer, model_name: str):
62
63
64
65
66
67
68
69
70
71
72
    input_texts = [
        "The product arrived on time and works perfectly",
        "I'm very satisfied with my purchase, would buy again",
        "The customer service was helpful and resolved my issue quickly",
        "This product broke after one week, terrible quality",
        "I'm very disappointed with this purchase, complete waste of money",
        "The customer service was rude and unhelpful",
    ]

    classification_response = requests.post(
        server.url_for("classify"),
73
        json={"model": model_name, "input": input_texts},
74
    )
75
    output = ClassificationResponse.model_validate(classification_response.json())
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

    assert len(output.data) == len(input_texts)
    for i, item in enumerate(output.data):
        assert item.index == i
        assert hasattr(item, "label")
        assert hasattr(item, "probs")
        assert len(item.probs) == item.num_classes
        assert item.label in ["Default", "Spoiled"]


@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_truncate_prompt_tokens(server: RemoteOpenAIServer, model_name: str):
    long_text = "hello " * 600

    classification_response = requests.post(
        server.url_for("classify"),
92
        json={"model": model_name, "input": long_text, "truncate_prompt_tokens": 5},
93
94
95
    )

    classification_response.raise_for_status()
96
    output = ClassificationResponse.model_validate(classification_response.json())
97
98
99
100
101
102
103
104
105

    assert len(output.data) == 1
    assert output.data[0].index == 0
    assert hasattr(output.data[0], "probs")
    assert output.usage.prompt_tokens == 5
    assert output.usage.total_tokens == 5


@pytest.mark.parametrize("model_name", [MODEL_NAME])
106
107
108
def test_invalid_truncate_prompt_tokens_error(
    server: RemoteOpenAIServer, model_name: str
):
109
110
    classification_response = requests.post(
        server.url_for("classify"),
111
        json={"model": model_name, "input": "test", "truncate_prompt_tokens": 513},
112
113
114
115
    )

    error = classification_response.json()
    assert classification_response.status_code == 400
116
    assert "truncate_prompt_tokens" in error["error"]["message"]
117
118
119
120
121
122


@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_empty_input_error(server: RemoteOpenAIServer, model_name: str):
    classification_response = requests.post(
        server.url_for("classify"),
123
        json={"model": model_name, "input": ""},
124
125
126
127
    )

    error = classification_response.json()
    assert classification_response.status_code == 400
128
    assert "error" in error
129
130
131


@pytest.mark.parametrize("model_name", [MODEL_NAME])
132
def test_batch_classification_empty_list(server: RemoteOpenAIServer, model_name: str):
133
134
    classification_response = requests.post(
        server.url_for("classify"),
135
        json={"model": model_name, "input": []},
136
137
    )
    classification_response.raise_for_status()
138
    output = ClassificationResponse.model_validate(classification_response.json())
139
140
141
142

    assert output.object == "list"
    assert isinstance(output.data, list)
    assert len(output.data) == 0
143
144
145
146
147
148


@pytest.mark.asyncio
async def test_invocations(server: RemoteOpenAIServer):
    request_args = {
        "model": MODEL_NAME,
149
        "input": "This product was excellent and exceeded my expectations",
150
151
    }

152
153
154
    classification_response = requests.post(
        server.url_for("classify"), json=request_args
    )
155
156
    classification_response.raise_for_status()

157
158
159
    invocation_response = requests.post(
        server.url_for("invocations"), json=request_args
    )
160
161
162
163
164
165
    invocation_response.raise_for_status()

    classification_output = classification_response.json()
    invocation_output = invocation_response.json()

    assert classification_output.keys() == invocation_output.keys()
166
    for classification_data, invocation_data in zip(
167
168
        classification_output["data"], invocation_output["data"]
    ):
169
170
        assert classification_data.keys() == invocation_data.keys()
        assert classification_data["probs"] == pytest.approx(
171
172
            invocation_data["probs"], rel=0.01
        )
173
174
175
176


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
177
async def test_use_activation(server: RemoteOpenAIServer, model_name: str):
178
179
    input_text = ["This product was excellent and exceeded my expectations"]

180
    async def get_outputs(use_activation):
181
182
        response = requests.post(
            server.url_for("classify"),
183
184
185
186
187
            json={
                "model": model_name,
                "input": input_text,
                "use_activation": use_activation,
            },
188
        )
189
        outputs = response.json()
190
        return torch.tensor([x["probs"] for x in outputs["data"]])
191

192
193
194
    default = await get_outputs(use_activation=None)
    w_activation = await get_outputs(use_activation=True)
    wo_activation = await get_outputs(use_activation=False)
195

196
197
198
199
200
201
202
203
204
    assert torch.allclose(default, w_activation, atol=1e-2), (
        "Default should use activation."
    )
    assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
        "wo_activation should not use activation."
    )
    assert torch.allclose(F.softmax(wo_activation, dim=-1), w_activation, atol=1e-2), (
        "w_activation should be close to activation(wo_activation)."
    )
205
206
207
208


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
209
async def test_score(server: RemoteOpenAIServer, model_name: str):
210
211
212
213
214
215
216
217
218
219
220
221
222
223
    # score api is only enabled for num_labels == 1.
    response = requests.post(
        server.url_for("score"),
        json={
            "model": model_name,
            "text_1": "ping",
            "text_2": "pong",
        },
    )
    assert response.json()["error"]["type"] == "BadRequestError"


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
224
async def test_rerank(server: RemoteOpenAIServer, model_name: str):
225
226
227
228
229
230
231
232
233
234
    # rerank api is only enabled for num_labels == 1.
    response = requests.post(
        server.url_for("rerank"),
        json={
            "model": model_name,
            "query": "ping",
            "documents": ["pong"],
        },
    )
    assert response.json()["error"]["type"] == "BadRequestError"
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_pooling_classify(server: RemoteOpenAIServer, model_name: str):
    input_text = "This product was excellent and exceeded my expectations"
    response = requests.post(
        server.url_for("pooling"),
        json={
            "model": model_name,
            "input": input_text,
            "encoding_format": "float",
            "task": "classify",
        },
    )
    poolings = PoolingResponse.model_validate(response.json())
    assert len(poolings.data) == 1
    assert len(poolings.data[0].data) == 2


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: str):
    task = "token_classify"
259
    input_text = ["This product was excellent and exceeded my expectations"]
260
261
262
263
    response = requests.post(
        server.url_for("pooling"),
        json={
            "model": model_name,
264
            "input": input_text,
265
266
267
268
            "encoding_format": "float",
            "task": task,
        },
    )
269
270
271
272
    poolings = PoolingResponse.model_validate(response.json())
    assert len(poolings.data) == 1
    assert len(poolings.data[0].data) == 8
    assert len(poolings.data[0].data[0]) == 2
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("task", ["embed", "token_embed", "plugin"])
async def test_pooling_not_supported(
    server: RemoteOpenAIServer, model_name: str, task: str
):
    response = requests.post(
        server.url_for("pooling"),
        json={
            "model": model_name,
            "input": "test",
            "encoding_format": "float",
            "task": task,
        },
    )
    assert response.json()["error"]["type"] == "BadRequestError"
    assert response.json()["error"]["message"].startswith(
        f"Task {task} is not supported"
    )