Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
conftest.py 347 Bytes
Newer Older
Ben Browning's avatar
[Chore]: Stream tokens vs characters in tool call parser tests (#26513)  
Ben Browning committed
1
2
3
4
5
6
7
8
9
10
11
12 
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import pytest
from transformers import AutoTokenizer

from vllm.transformers_utils.tokenizer import AnyTokenizer


@pytest.fixture(scope="function")
def default_tokenizer() -> AnyTokenizer:
    return AutoTokenizer.from_pretrained("gpt2")