Tiny move files to utils folder (#11166)

python/sglang/test/runners.py

@@ -30,8 +30,8 @@ from transformers import (
 )
 
 from sglang.srt.entrypoints.engine import Engine
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import load_image
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import DEFAULT_PORT_FOR_SRT_TEST_RUNNER, calculate_rouge_l
 
 DEFAULT_PROMPTS = [

python/sglang/test/test_programs.py

@@ -551,7 +551,7 @@ def test_gen_min_new_tokens():
     We verify that the number of tokens in the answer is >= the min_tokens threshold.
     """
     import sglang as sgl
-    from sglang.srt.hf_transformers_utils import get_tokenizer
+    from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 
     model_path = sgl.global_config.default_backend.endpoint.get_model_name()
     MIN_TOKENS, MAX_TOKENS = 64, 128

python/sglang/test/test_utils.py

@@ -921,7 +921,7 @@ def run_score_benchmark(
     async def _run_benchmark():
 
         # Load tokenizer for generating test data
-        from sglang.srt.hf_transformers_utils import get_tokenizer
+        from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 
         tokenizer = get_tokenizer(model)
 

scripts/playground/load_tokenizer.py

 import argparse
 import code
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()

scripts/playground/reference_hf.py

@@ -38,7 +38,7 @@ from transformers import (
     AutoProcessor,
 )
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 
 
 @torch.no_grad()

test/srt/openai_server/basic/test_openai_server.py

@@ -13,8 +13,8 @@ import numpy as np
 import openai
 import requests
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.runners import TEST_RERANK_QUERY_DOCS
 from sglang.test.test_utils import (
     DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST,

test/srt/openai_server/features/test_enable_thinking.py

@@ -16,8 +16,8 @@ import unittest
 import openai
 import requests
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,

test/srt/openai_server/features/test_openai_server_ebnf.py

@@ -2,8 +2,8 @@ import re
 
 import openai
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,

test/srt/openai_server/features/test_openai_server_hidden_states.py

@@ -8,8 +8,8 @@ import numpy as np
 import openai
 import torch
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
     DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,

test/srt/openai_server/function_call/test_openai_function_calling.py

@@ -4,8 +4,8 @@ import unittest
 
 import openai
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,

test/srt/openai_server/function_call/test_tool_choice.py

@@ -12,8 +12,8 @@ import unittest
 
 import openai
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,

test/srt/openai_server/validation/test_large_max_new_tokens.py

@@ -9,8 +9,8 @@ from concurrent.futures import ThreadPoolExecutor
 
 import openai
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,

test/srt/openai_server/validation/test_openai_server_ignore_eos.py

 import openai
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,

test/srt/rl/test_verl_engine_2_gpu.py

@@ -19,8 +19,8 @@ from torch.distributed.fsdp.api import (
 from transformers import AutoModelForCausalLM
 
 from sglang.srt.entrypoints.verl_engine import VerlEngine
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import is_port_available
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.runners import (
     HFRunner,
     SRTRunner,

test/srt/rl/test_verl_engine_4_gpu.py

@@ -19,8 +19,8 @@ from torch.distributed.fsdp.api import (
 from transformers import AutoModelForCausalLM
 
 from sglang.srt.entrypoints.verl_engine import VerlEngine
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import is_port_available
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.runners import (
     HFRunner,
     SRTRunner,

test/srt/test_bench_serving.py

@@ -4,7 +4,7 @@ import unittest
 
 import requests
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
     DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,

test/srt/test_eagle_infer_a.py

@@ -4,8 +4,8 @@ import requests
 import torch
 
 import sglang as sgl
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
     DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,

test/srt/test_fim_completion.py

@@ -2,8 +2,8 @@ import unittest
 
 import openai
 
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,

test/srt/test_forward_split_prefill.py

@@ -13,13 +13,13 @@ import numpy as np
 import torch
 
 from sglang.srt.configs.model_config import ModelConfig
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
 from sglang.srt.model_executor.forward_batch_info import ForwardBatch
 from sglang.srt.model_executor.model_runner import ModelRunner
 from sglang.srt.sampling.sampling_params import SamplingParams
 from sglang.srt.server_args import PortArgs, ServerArgs
 from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase
 
 

test/srt/test_function_call_parser.py

@@ -15,7 +15,7 @@ from sglang.srt.function_call.mistral_detector import MistralDetector
 from sglang.srt.function_call.pythonic_detector import PythonicDetector
 from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
 from sglang.srt.function_call.qwen25_detector import Qwen25Detector
-from sglang.srt.hf_transformers_utils import get_tokenizer
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST