"vllm/vscode:/vscode.git/clone" did not exist on "b5708117060460fcc6ba6f58e0669d3c46d6339e"
test_oot_registration.py 1.85 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
import os
from ...utils import VLLM_PATH, RemoteOpenAIServer, models_path_prefix, envs
6
7
8
9

chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
assert chatml_jinja_path.exists()

10

11
12
def run_and_test_dummy_opt_api_server(model, tp=1):
    # the model is registered through the plugin
zhuwenwen's avatar
zhuwenwen committed
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
    if envs.VLLM_USE_TRITON_FLASH_ATTN:
        server_args = [
            "--gpu-memory-utilization",
            "0.10",
            "--dtype",
            "float32",
            "--chat-template",
            str(chatml_jinja_path),
            "--load-format",
            "dummy",
            "-tp",
            f"{tp}",
        ]
    else:
        server_args = [
            "--gpu-memory-utilization",
            "0.10",
            "--dtype",
            "float16",
            "--chat-template",
            str(chatml_jinja_path),
            "--load-format",
            "dummy",
            "-tp",
            f"{tp}",
38
    ]
39
40
41
42
43
44
45
46
47
48
49
50
    with RemoteOpenAIServer(model, server_args) as server:
        client = server.get_client()
        completion = client.chat.completions.create(
            model=model,
            messages=[{
                "role": "system",
                "content": "You are a helpful assistant."
            }, {
                "role": "user",
                "content": "Hello!"
            }],
            temperature=0,
51
        )
52
53
        generated_text = completion.choices[0].message.content
        assert generated_text is not None
zhuwenwen's avatar
zhuwenwen committed
54
        # make sure only the first token is generated
55
56
57
        rest = generated_text.replace("<s>", "")
        assert rest == ""

58

59
def test_oot_registration_for_api_server(dummy_opt_path: str):
60
    dummy_opt_path = os.path.join(models_path_prefix, "facebook/opt-125m")
61
    run_and_test_dummy_opt_api_server(dummy_opt_path)