envs.py 71 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import functools
5
import hashlib
6
import json
7
import os
8
import sys
9
import tempfile
10
11
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Literal
12
13
14

if TYPE_CHECKING:
    VLLM_HOST_IP: str = ""
15
    VLLM_PORT: int | None = None
16
    VLLM_RPC_BASE_PATH: str = tempfile.gettempdir()
17
    VLLM_USE_MODELSCOPE: bool = False
18
    VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
19
20
    VLLM_NCCL_SO_PATH: str | None = None
    LD_LIBRARY_PATH: str | None = None
21
    VLLM_USE_TRITON_FLASH_ATTN: bool = True
22
    VLLM_V1_USE_PREFILL_DECODE_ATTENTION: bool = False
23
    VLLM_FLASH_ATTN_VERSION: int | None = None
24
    LOCAL_RANK: int = 0
25
    CUDA_VISIBLE_DEVICES: str | None = None
26
    VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60
27
    VLLM_API_KEY: str | None = None
28
    VLLM_DEBUG_LOG_API_SERVER_RESPONSE: bool = False
29
30
31
32
    S3_ACCESS_KEY_ID: str | None = None
    S3_SECRET_ACCESS_KEY: str | None = None
    S3_ENDPOINT_URL: str | None = None
    VLLM_MODEL_REDIRECT_PATH: str | None = None
33
34
    VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm")
    VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm")
35
36
    VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai"
    VLLM_NO_USAGE_STATS: bool = False
37
    VLLM_DISABLE_FLASHINFER_PREFILL: bool = False
38
39
40
    VLLM_DO_NOT_TRACK: bool = False
    VLLM_USAGE_SOURCE: str = ""
    VLLM_CONFIGURE_LOGGING: int = 1
41
    VLLM_LOGGING_LEVEL: str = "INFO"
42
    VLLM_LOGGING_PREFIX: str = ""
43
    VLLM_LOGGING_STREAM: str = "ext://sys.stdout"
44
    VLLM_LOGGING_CONFIG_PATH: str | None = None
45
    VLLM_LOG_STATS_INTERVAL: float = 10.0
46
    VLLM_TRACE_FUNCTION: int = 0
47
48
49
50
    VLLM_ATTENTION_BACKEND: str | None = None
    VLLM_USE_FLASHINFER_SAMPLER: bool | None = None
    VLLM_PP_LAYER_PARTITION: str | None = None
    VLLM_CPU_KVCACHE_SPACE: int | None = 0
51
    VLLM_CPU_OMP_THREADS_BIND: str = ""
52
    VLLM_CPU_NUM_OF_RESERVED_CPU: int | None = None
53
    VLLM_CPU_MOE_PREPACK: bool = True
54
    VLLM_CPU_SGL_KERNEL: bool = False
55
    VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
56
    VLLM_XLA_CHECK_RECOMPILATION: bool = False
57
    VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
58
    VLLM_ENABLE_FUSED_MOE_ACTIVATION_CHUNKING: bool = True
59
    VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE: Literal["auto", "nccl", "shm"] = "auto"
60
    VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False
61
    VLLM_USE_RAY_WRAPPED_PP_COMM: bool = True
62
    VLLM_XLA_USE_SPMD: bool = False
63
    VLLM_WORKER_MULTIPROC_METHOD: Literal["fork", "spawn"] = "fork"
64
    VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
65
    VLLM_ASSETS_CACHE_MODEL_CLEAN: bool = False
66
    VLLM_IMAGE_FETCH_TIMEOUT: int = 5
67
    VLLM_VIDEO_FETCH_TIMEOUT: int = 30
68
    VLLM_AUDIO_FETCH_TIMEOUT: int = 10
69
    VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
70
    VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
71
    VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
72
    VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
73
    VLLM_MM_INPUT_CACHE_GIB: int = 4
74
    VLLM_TARGET_DEVICE: str = "cuda"
75
    VLLM_MAIN_CUDA_VERSION: str = "12.8"
76
77
    MAX_JOBS: str | None = None
    NVCC_THREADS: str | None = None
78
    VLLM_USE_PRECOMPILED: bool = False
79
    VLLM_DOCKER_BUILD_CONTEXT: bool = False
80
    VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL: bool = False
81
    VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
82
    CMAKE_BUILD_TYPE: Literal["Debug", "Release", "RelWithDebInfo"] | None = None
83
    VERBOSE: bool = False
84
    VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
85
    VLLM_RPC_TIMEOUT: int = 10000  # ms
86
    VLLM_HTTP_TIMEOUT_KEEP_ALIVE: int = 5  # seconds
87
88
89
    VLLM_PLUGINS: list[str] | None = None
    VLLM_LORA_RESOLVER_CACHE_DIR: str | None = None
    VLLM_TORCH_PROFILER_DIR: str | None = None
90
91
    VLLM_TORCH_PROFILER_RECORD_SHAPES: bool = False
    VLLM_TORCH_PROFILER_WITH_PROFILE_MEMORY: bool = False
92
93
    VLLM_USE_AOT_COMPILE: bool = False
    VLLM_FORCE_AOT_LOAD: bool = False
94
95
    VLLM_TORCH_PROFILER_WITH_STACK: bool = True
    VLLM_TORCH_PROFILER_WITH_FLOPS: bool = False
96
    VLLM_USE_TRITON_AWQ: bool = False
97
    VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
98
    VLLM_SKIP_P2P_CHECK: bool = False
99
    VLLM_DISABLED_KERNELS: list[str] = []
100
    VLLM_DISABLE_PYNCCL: bool = False
101
    VLLM_USE_V1: bool = True
102
    VLLM_ROCM_USE_AITER: bool = False
103
    VLLM_ROCM_USE_AITER_PAGED_ATTN: bool = False
104
    VLLM_ROCM_USE_AITER_LINEAR: bool = True
105
    VLLM_ROCM_USE_AITER_MOE: bool = True
106
    VLLM_ROCM_USE_AITER_RMSNORM: bool = True
107
    VLLM_ROCM_USE_AITER_MLA: bool = True
108
    VLLM_ROCM_USE_AITER_MHA: bool = True
109
110
    VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
    VLLM_ROCM_USE_TRITON_ROPE: bool = False
111
    VLLM_ROCM_USE_AITER_FP8BMM: bool = True
112
    VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: bool = False
113
    VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: bool = True
114
    VLLM_ROCM_USE_SKINNY_GEMM: bool = True
115
    VLLM_ROCM_FP8_PADDING: bool = True
116
    VLLM_ROCM_MOE_PADDING: bool = True
117
    VLLM_ROCM_CUSTOM_PAGED_ATTN: bool = True
118
    VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
119
    VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
120
    VLLM_DISABLE_COMPILE_CACHE: bool = False
121
    Q_SCALE_CONSTANT: int = 200
122
123
    K_SCALE_CONSTANT: int = 200
    V_SCALE_CONSTANT: int = 100
124
    VLLM_SERVER_DEV_MODE: bool = False
125
    VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
126
    VLLM_MLA_DISABLE: bool = False
127
    VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH: int = 32
128
129
    VLLM_RAY_PER_WORKER_GPUS: float = 1.0
    VLLM_RAY_BUNDLE_INDICES: str = ""
130
    VLLM_CUDART_SO_PATH: str | None = None
131
    VLLM_DP_RANK: int = 0
132
    VLLM_DP_RANK_LOCAL: int = -1
133
    VLLM_DP_SIZE: int = 1
134
    VLLM_USE_STANDALONE_COMPILE: bool = True
135
136
    VLLM_DP_MASTER_IP: str = ""
    VLLM_DP_MASTER_PORT: int = 0
137
    VLLM_MOE_DP_CHUNK_SIZE: int = 256
138
    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: bool = False
139
    VLLM_RAY_DP_PACK_STRATEGY: Literal["strict", "fill", "span"] = "strict"
140
    VLLM_MARLIN_USE_ATOMIC_ADD: bool = False
141
    VLLM_MXFP4_USE_MARLIN: bool | None = None
142
    VLLM_V1_USE_OUTLINES_CACHE: bool = False
143
    VLLM_TPU_BUCKET_PADDING_GAP: int = 0
144
    VLLM_TPU_MOST_MODEL_LEN: int | None = None
145
    VLLM_TPU_USING_PATHWAYS: bool = False
146
    VLLM_USE_DEEP_GEMM: bool = True
147
    VLLM_USE_DEEP_GEMM_E8M0: bool = True
148
149
150
151
152
    VLLM_DEEP_GEMM_WARMUP: Literal[
        "skip",
        "full",
        "relax",
    ] = "relax"
153
    VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
154
    VLLM_USE_FLASHINFER_MOE_FP16: bool = False
155
156
    VLLM_USE_FLASHINFER_MOE_FP8: bool = False
    VLLM_USE_FLASHINFER_MOE_FP4: bool = False
157
    VLLM_FLASHINFER_MOE_BACKEND: Literal["throughput", "latency"] = "throughput"
158
    VLLM_XGRAMMAR_CACHE_MB: int = 0
159
    VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256
160
    VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False
Robert Shaw's avatar
Robert Shaw committed
161
    VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
162
    VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
163
164
165
166
167
168
169
170
    VLLM_ALL2ALL_BACKEND: Literal[
        "naive",
        "pplx",
        "deepep_high_throughput",
        "deepep_low_latency",
        "allgather_reducescatter",
        "flashinfer_all2allv",
    ] = "allgather_reducescatter"
171
    VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
172
    VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
173
    VLLM_SLEEP_WHEN_IDLE: bool = False
174
    VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
175
    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
176
    VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
177
    VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
178
    VLLM_USE_NVFP4_CT_EMULATIONS: bool = False
179
180
181
    VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
        "FP", "INT8", "INT6", "INT4", "NONE"
    ] = "NONE"
182
    VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16: bool = True
183
    VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: int | None = None
184
    VLLM_NIXL_ABORT_REQUEST_TIMEOUT: int = 480
185
    VLLM_USE_CUDNN_PREFILL: bool = False
186
    VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: bool = False
187
    VLLM_ENABLE_CUDAGRAPH_GC: bool = False
188
    VLLM_LOOPBACK_IP: str = ""
189
    VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = False
190
    VLLM_ENABLE_RESPONSES_API_STORE: bool = False
191
    VLLM_USE_TRTLLM_ATTENTION: str | None = None
192
    VLLM_NVFP4_GEMM_BACKEND: str | None = None
193
    VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION: bool = False
194
    VLLM_HAS_FLASHINFER_CUBIN: bool = False
195
196
    VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False
    VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False
xiao-llm's avatar
xiao-llm committed
197
    VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
198
    VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False
199
    VLLM_ALLREDUCE_USE_SYMM_MEM: bool = False
200
    VLLM_TUNED_CONFIG_FOLDER: str | None = None
201
    VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
202
    VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
203
    VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False
204
    VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
205
    VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False
206
    VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True
207
    VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME: str = "VLLM_OBJECT_STORAGE_SHM_BUFFER"
208
    VLLM_DEEPEP_BUFFER_SIZE_MB: int = 1024
209
210
    VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE: bool = False
    VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL: bool = False
211
    VLLM_DBO_COMM_SMS: int = 20
212
213
    VLLM_PATTERN_MATCH_DEBUG: str | None = None
    VLLM_DEBUG_DUMP_PATH: str | None = None
214
215
    VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE: bool = True
    VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True
216
    VLLM_USE_NCCL_SYMM_MEM: bool = False
217
    VLLM_NCCL_INCLUDE_PATH: str | None = None
218
    VLLM_USE_FBGEMM: bool = False
219
    VLLM_GC_DEBUG: str = ""
220
    VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
221

222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

def get_default_cache_root():
    return os.getenv(
        "XDG_CACHE_HOME",
        os.path.join(os.path.expanduser("~"), ".cache"),
    )


def get_default_config_root():
    return os.getenv(
        "XDG_CONFIG_HOME",
        os.path.join(os.path.expanduser("~"), ".config"),
    )


237
def maybe_convert_int(value: str | None) -> int | None:
238
239
240
241
242
    if value is None:
        return None
    return int(value)


243
def maybe_convert_bool(value: str | None) -> bool | None:
244
245
246
247
248
    if value is None:
        return None
    return bool(int(value))


249
250
251
252
def disable_compile_cache() -> bool:
    return bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0")))


253
def use_aot_compile() -> bool:
254
    from vllm.utils.torch_utils import is_torch_equal_or_newer
255

256
257
258
259
260
261
    default_value = (
        "1"
        if is_torch_equal_or_newer("2.10.0.dev") and not disable_compile_cache()
        else "0"
    )

262
263
264
    return os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1"


265
def env_with_choices(
266
    env_name: str,
267
268
    default: str | None,
    choices: list[str] | Callable[[], list[str]],
269
    case_sensitive: bool = True,
270
) -> Callable[[], str | None]:
271
272
    """
    Create a lambda that validates environment variable against allowed choices
273

274
275
276
277
278
    Args:
        env_name: Name of the environment variable
        default: Default value if not set (can be None)
        choices: List of valid string options or callable that returns list
        case_sensitive: Whether validation should be case sensitive
279

280
281
282
283
    Returns:
        Lambda function for environment_variables dict
    """

284
    def _get_validated_env() -> str | None:
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
        value = os.getenv(env_name)
        if value is None:
            return default

        # Resolve choices if it's a callable (for lazy loading)
        actual_choices = choices() if callable(choices) else choices

        if not case_sensitive:
            check_value = value.lower()
            check_choices = [choice.lower() for choice in actual_choices]
        else:
            check_value = value
            check_choices = actual_choices

        if check_value not in check_choices:
300
301
302
303
            raise ValueError(
                f"Invalid value '{value}' for {env_name}. "
                f"Valid options: {actual_choices}."
            )
304
305
306
307
308
309

        return value

    return _get_validated_env


310
def env_list_with_choices(
311
312
    env_name: str,
    default: list[str],
313
    choices: list[str] | Callable[[], list[str]],
314
315
    case_sensitive: bool = True,
) -> Callable[[], list[str]]:
316
    """
317
    Create a lambda that validates environment variable
318
    containing comma-separated values against allowed choices
319

320
321
322
323
324
    Args:
        env_name: Name of the environment variable
        default: Default list of values if not set
        choices: List of valid string options or callable that returns list
        case_sensitive: Whether validation should be case sensitive
325

326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
    Returns:
        Lambda function for environment_variables
        dict that returns list of strings
    """

    def _get_validated_env_list() -> list[str]:
        value = os.getenv(env_name)
        if value is None:
            return default

        # Split comma-separated values and strip whitespace
        values = [v.strip() for v in value.split(",") if v.strip()]

        if not values:
            return default

        # Resolve choices if it's a callable (for lazy loading)
        actual_choices = choices() if callable(choices) else choices

        # Validate each value
        for val in values:
            if not case_sensitive:
                check_value = val.lower()
                check_choices = [choice.lower() for choice in actual_choices]
            else:
                check_value = val
                check_choices = actual_choices

            if check_value not in check_choices:
355
356
357
358
                raise ValueError(
                    f"Invalid value '{val}' in {env_name}. "
                    f"Valid options: {actual_choices}."
                )
359
360
361
362
363
364

        return values

    return _get_validated_env_list


365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
def env_set_with_choices(
    env_name: str,
    default: list[str],
    choices: list[str] | Callable[[], list[str]],
    case_sensitive: bool = True,
) -> Callable[[], set[str]]:
    """
    Creates a lambda which that validates environment variable
    containing comma-separated values against allowed choices which
    returns choices as a set.
    """

    def _get_validated_env_set() -> set[str]:
        return set(env_list_with_choices(env_name, default, choices, case_sensitive)())

    return _get_validated_env_set


383
def get_vllm_port() -> int | None:
384
    """Get the port from VLLM_PORT environment variable.
385

386
387
    Returns:
        The port number as an integer if VLLM_PORT is set, None otherwise.
388

389
390
391
    Raises:
        ValueError: If VLLM_PORT is a URI, suggest k8s service discovery issue.
    """
392
    if "VLLM_PORT" not in os.environ:
393
394
        return None

395
    port = os.getenv("VLLM_PORT", "0")
396
397
398
399
400

    try:
        return int(port)
    except ValueError as err:
        from urllib.parse import urlparse
401

402
403
404
405
406
407
408
        parsed = urlparse(port)
        if parsed.scheme:
            raise ValueError(
                f"VLLM_PORT '{port}' appears to be a URI. "
                "This may be caused by a Kubernetes service discovery issue,"
                "check the warning in: https://docs.vllm.ai/en/stable/serving/env_vars.html"
            ) from None
409
        raise ValueError(f"VLLM_PORT '{port}' must be a valid integer") from err
410
411


412
413
414
# The begin-* and end* here are used by the documentation generator
# to extract the used env vars.

415
# --8<-- [start:env-vars-definition]
416

417
environment_variables: dict[str, Callable[[], Any]] = {
418
    # ================== Installation Time Env Vars ==================
419
    # Target device of vLLM, supporting [cuda (by default),
420
    # rocm, cpu]
421
    "VLLM_TARGET_DEVICE": lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda").lower(),
422
423
    # Main CUDA version of vLLM, supporting [12.6, 12.8, 12.9],
    # 12.8 is the default. This follows PyTorch but can be overridden.
424
425
    "VLLM_MAIN_CUDA_VERSION": lambda: os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower()
    or "12.8",
426
427
    # Maximum number of compilation jobs to run in parallel.
    # By default this is the number of CPUs
428
    "MAX_JOBS": lambda: os.getenv("MAX_JOBS", None),
429
430
431
    # Number of threads to use for nvcc
    # By default this is 1.
    # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
432
    "NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None),
433
    # If set, vllm will use precompiled binaries (*.so)
434
435
436
437
438
    "VLLM_USE_PRECOMPILED": lambda: os.environ.get("VLLM_USE_PRECOMPILED", "")
    .strip()
    .lower()
    in ("1", "true")
    or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION")),
439
440
    # Used to mark that setup.py is running in a Docker build context,
    # in order to force the use of precompiled binaries.
441
442
443
444
    "VLLM_DOCKER_BUILD_CONTEXT": lambda: os.environ.get("VLLM_DOCKER_BUILD_CONTEXT", "")
    .strip()
    .lower()
    in ("1", "true"),
445
446
    # Whether to force using nightly wheel in python build.
    # This is used for testing the nightly wheel in python build.
447
448
449
    "VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL": lambda: bool(
        int(os.getenv("VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL", "0"))
    ),
450
451
452
    # CMake build type
    # If not set, defaults to "Debug" or "RelWithDebInfo"
    # Available options: "Debug", "Release", "RelWithDebInfo"
453
454
455
    "CMAKE_BUILD_TYPE": env_with_choices(
        "CMAKE_BUILD_TYPE", None, ["Debug", "Release", "RelWithDebInfo"]
    ),
456
    # If set, vllm will print verbose logs during installation
457
    "VERBOSE": lambda: bool(int(os.getenv("VERBOSE", "0"))),
458
    # Root directory for vLLM configuration files
459
    # Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set
460
461
462
    # Note that this not only affects how vllm finds its configuration files
    # during runtime, but also affects how vllm installs its configuration
    # files during **installation**.
463
    "VLLM_CONFIG_ROOT": lambda: os.path.expanduser(
464
465
466
        os.getenv(
            "VLLM_CONFIG_ROOT",
            os.path.join(get_default_config_root(), "vllm"),
467
468
        )
    ),
469
    # ================== Runtime Env Vars ==================
470
    # Root directory for vLLM cache files
471
    # Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set
472
    "VLLM_CACHE_ROOT": lambda: os.path.expanduser(
473
474
475
        os.getenv(
            "VLLM_CACHE_ROOT",
            os.path.join(get_default_cache_root(), "vllm"),
476
477
        )
    ),
478
479
480
481
    # used in distributed environment to determine the ip address
    # of the current node, when the node has multiple network interfaces.
    # If you are using multi-node inference, you should set this differently
    # on each node.
482
    "VLLM_HOST_IP": lambda: os.getenv("VLLM_HOST_IP", ""),
483
    # used in distributed environment to manually set the communication port
484
485
486
    # Note: if VLLM_PORT is set, and some code asks for multiple ports, the
    # VLLM_PORT will be used as the first port, and the rest will be generated
    # by incrementing the VLLM_PORT value.
487
    "VLLM_PORT": get_vllm_port,
488
489
    # path used for ipc when the frontend api server is running in
    # multi-processing mode to communicate with the backend engine process.
490
491
492
    "VLLM_RPC_BASE_PATH": lambda: os.getenv(
        "VLLM_RPC_BASE_PATH", tempfile.gettempdir()
    ),
493
494
    # If true, will load models from ModelScope instead of Hugging Face Hub.
    # note that the value is true or false, not numbers
495
496
497
498
    "VLLM_USE_MODELSCOPE": lambda: os.environ.get(
        "VLLM_USE_MODELSCOPE", "False"
    ).lower()
    == "true",
499
    # Interval in seconds to log a warning message when the ring buffer is full
500
501
502
    "VLLM_RINGBUFFER_WARNING_INTERVAL": lambda: int(
        os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60")
    ),
503
504
    # path to cudatoolkit home directory, under which should be bin, include,
    # and lib directories.
505
    "CUDA_HOME": lambda: os.environ.get("CUDA_HOME", None),
506
507
    # Path to the NCCL library file. It is needed because nccl>=2.19 brought
    # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234
508
    "VLLM_NCCL_SO_PATH": lambda: os.environ.get("VLLM_NCCL_SO_PATH", None),
509
510
    # when `VLLM_NCCL_SO_PATH` is not set, vllm will try to find the nccl
    # library file in the locations specified by `LD_LIBRARY_PATH`
511
    "LD_LIBRARY_PATH": lambda: os.environ.get("LD_LIBRARY_PATH", None),
512
    # flag to control if vllm should use triton flash attention
513
514
515
    "VLLM_USE_TRITON_FLASH_ATTN": lambda: (
        os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in ("true", "1")
    ),
516
517
    # Use separate prefill and decode kernels for V1 attention instead of
    # the unified triton kernel.
518
519
520
521
    "VLLM_V1_USE_PREFILL_DECODE_ATTENTION": lambda: (
        os.getenv("VLLM_V1_USE_PREFILL_DECODE_ATTENTION", "False").lower()
        in ("true", "1")
    ),
522
523
    # Force vllm to use a specific flash-attention version (2 or 3), only valid
    # when using the flash-attention backend.
524
525
526
    "VLLM_FLASH_ATTN_VERSION": lambda: maybe_convert_int(
        os.environ.get("VLLM_FLASH_ATTN_VERSION", None)
    ),
527
    # Feature flag to enable/disable Inductor standalone compile.
528
529
    # In torch <= 2.7 we ignore this flag; in torch >= 2.9 this is
    # enabled by default.
530
    "VLLM_USE_STANDALONE_COMPILE": lambda: os.environ.get(
531
        "VLLM_USE_STANDALONE_COMPILE", "1"
532
533
    )
    == "1",
534
535
    # Debug pattern matching inside custom passes.
    # Should be set to the fx.Node name (e.g. 'getitem_34' or 'scaled_mm_3').
536
537
538
    "VLLM_PATTERN_MATCH_DEBUG": lambda: os.environ.get(
        "VLLM_PATTERN_MATCH_DEBUG", None
    ),
539
540
    # Dump fx graphs to the given directory.
    # It will override CompilationConfig.debug_dump_path if set.
541
    "VLLM_DEBUG_DUMP_PATH": lambda: os.environ.get("VLLM_DEBUG_DUMP_PATH", None),
542
543
544
545
546
547
548
549
    # Feature flag to enable/disable AOT compilation. This will ensure
    # compilation is done in warmup phase and the compilation will be
    # reused in subsequent calls.
    "VLLM_USE_AOT_COMPILE": use_aot_compile,
    # Force vllm to always load AOT compiled models from disk. Failure
    # to load will result in a hard error when this is enabled.
    # Will be ignored when VLLM_USE_AOT_COMPILE is disabled.
    "VLLM_FORCE_AOT_LOAD": lambda: os.environ.get("VLLM_FORCE_AOT_LOAD", "0") == "1",
550
551
    # local rank of the process in the distributed setting, used to determine
    # the GPU device id
552
    "LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")),
553
    # used to control the visible devices in the distributed setting
554
    "CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
555
    # timeout for each iteration in the engine
556
557
558
    "VLLM_ENGINE_ITERATION_TIMEOUT_S": lambda: int(
        os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60")
    ),
559
    # API key for vLLM API server
560
    "VLLM_API_KEY": lambda: os.environ.get("VLLM_API_KEY", None),
561
    # Whether to log responses from API Server for debugging
562
563
564
565
    "VLLM_DEBUG_LOG_API_SERVER_RESPONSE": lambda: os.environ.get(
        "VLLM_DEBUG_LOG_API_SERVER_RESPONSE", "False"
    ).lower()
    == "true",
566
    # S3 access information, used for tensorizer to load model from S3
567
568
569
    "S3_ACCESS_KEY_ID": lambda: os.environ.get("S3_ACCESS_KEY_ID", None),
    "S3_SECRET_ACCESS_KEY": lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None),
    "S3_ENDPOINT_URL": lambda: os.environ.get("S3_ENDPOINT_URL", None),
570
    # Usage stats collection
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
    "VLLM_USAGE_STATS_SERVER": lambda: os.environ.get(
        "VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai"
    ),
    "VLLM_NO_USAGE_STATS": lambda: os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1",
    "VLLM_DISABLE_FLASHINFER_PREFILL": lambda: os.environ.get(
        "VLLM_DISABLE_FLASHINFER_PREFILL", "0"
    )
    == "1",
    "VLLM_DO_NOT_TRACK": lambda: (
        os.environ.get("VLLM_DO_NOT_TRACK", None)
        or os.environ.get("DO_NOT_TRACK", None)
        or "0"
    )
    == "1",
    "VLLM_USAGE_SOURCE": lambda: os.environ.get("VLLM_USAGE_SOURCE", "production"),
586
587
588
589
    # Logging configuration
    # If set to 0, vllm will not configure logging
    # If set to 1, vllm will configure logging using the default configuration
    #    or the configuration file specified by VLLM_LOGGING_CONFIG_PATH
590
591
    "VLLM_CONFIGURE_LOGGING": lambda: int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")),
    "VLLM_LOGGING_CONFIG_PATH": lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"),
592
    # this is used for configuring the default logging level
593
    "VLLM_LOGGING_LEVEL": lambda: os.getenv("VLLM_LOGGING_LEVEL", "INFO").upper(),
594
    # this is used for configuring the default logging stream
595
    "VLLM_LOGGING_STREAM": lambda: os.getenv("VLLM_LOGGING_STREAM", "ext://sys.stdout"),
596
    # if set, VLLM_LOGGING_PREFIX will be prepended to all log messages
597
    "VLLM_LOGGING_PREFIX": lambda: os.getenv("VLLM_LOGGING_PREFIX", ""),
598
599
    # If set, vllm will log stats at this interval in seconds
    # If not set, vllm will log stats every 10 seconds.
600
601
602
    "VLLM_LOG_STATS_INTERVAL": lambda: val
    if (val := float(os.getenv("VLLM_LOG_STATS_INTERVAL", "10."))) > 0.0
    else 10.0,
603
604
605
    # Trace function calls
    # If set to 1, vllm will trace function calls
    # Useful for debugging
606
    "VLLM_TRACE_FUNCTION": lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")),
607
    # Backend for attention computation
608
    # Example options:
609
610
611
    # - "TORCH_SDPA": use torch.nn.MultiheadAttention
    # - "FLASH_ATTN": use FlashAttention
    # - "XFORMERS": use XFormers
612
    # - "FLASHINFER": use flashinfer
613
    # - "FLASHMLA": use FlashMLA
614
    # - "FLASH_ATTN_MLA": use FlashAttention for MLA
615
616
    # - "FLASHINFER_MLA": use FlashInfer for MLA
    # - "CUTLASS_MLA": use CUTLASS for MLA
617
    # All possible options loaded dynamically from _Backend enum
618
619
620
621
622
623
624
625
626
    "VLLM_ATTENTION_BACKEND": env_with_choices(
        "VLLM_ATTENTION_BACKEND",
        None,
        lambda: list(
            __import__(
                "vllm.attention.backends.registry", fromlist=["_Backend"]
            )._Backend.__members__.keys()
        ),
    ),
627
    # If set, vllm will use flashinfer sampler
628
629
630
631
632
    "VLLM_USE_FLASHINFER_SAMPLER": lambda: bool(
        int(os.environ["VLLM_USE_FLASHINFER_SAMPLER"])
    )
    if "VLLM_USE_FLASHINFER_SAMPLER" in os.environ
    else None,
633
    # Pipeline stage partition strategy
634
    "VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
635
    # (CPU backend only) CPU key-value cache space.
636
    # default is None and will be set as 4 GB
637
638
639
    "VLLM_CPU_KVCACHE_SPACE": lambda: int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0"))
    if "VLLM_CPU_KVCACHE_SPACE" in os.environ
    else None,
640
641
    # (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
    # "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'.
642
    "VLLM_CPU_OMP_THREADS_BIND": lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "auto"),
643
644
    # (CPU backend only) CPU cores not used by OMP threads .
    # Those CPU cores will not be used by OMP threads of a rank.
645
646
647
648
649
    "VLLM_CPU_NUM_OF_RESERVED_CPU": lambda: int(
        os.getenv("VLLM_CPU_NUM_OF_RESERVED_CPU", "0")
    )
    if "VLLM_CPU_NUM_OF_RESERVED_CPU" in os.environ
    else None,
650
651
652
    # (CPU backend only) whether to use prepack for MoE layer. This will be
    # passed to ipex.llm.modules.GatedMLPMOE. On unsupported CPUs, you might
    # need to set this to "0" (False).
653
    "VLLM_CPU_MOE_PREPACK": lambda: bool(int(os.getenv("VLLM_CPU_MOE_PREPACK", "1"))),
654
    # (CPU backend only) whether to use SGL kernels, optimized for small batch.
655
    "VLLM_CPU_SGL_KERNEL": lambda: bool(int(os.getenv("VLLM_CPU_SGL_KERNEL", "0"))),
656
657
658
659
660
661
662
    # If the env var is set, Ray Compiled Graph uses the specified
    # channel type to communicate between workers belonging to
    # different pipeline-parallel stages.
    # Available options:
    # - "auto": use the default channel type
    # - "nccl": use NCCL for communication
    # - "shm": use shared memory and gRPC for communication
663
664
665
    "VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE": env_with_choices(
        "VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE", "auto", ["auto", "nccl", "shm"]
    ),
666
    # If the env var is set, it enables GPU communication overlap
667
    # (experimental feature) in Ray's Compiled Graph.
668
669
670
    "VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM": lambda: bool(
        int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "0"))
    ),
671
672
673
    # If the env var is set, it uses a Ray Communicator wrapping
    # vLLM's pipeline parallelism communicator to interact with Ray's
    # Compiled Graph. Otherwise, it uses Ray's NCCL communicator.
674
675
676
    "VLLM_USE_RAY_WRAPPED_PP_COMM": lambda: bool(
        int(os.getenv("VLLM_USE_RAY_WRAPPED_PP_COMM", "1"))
    ),
677
678
    # Use dedicated multiprocess context for workers.
    # Both spawn and fork work
679
680
681
    "VLLM_WORKER_MULTIPROC_METHOD": env_with_choices(
        "VLLM_WORKER_MULTIPROC_METHOD", "fork", ["spawn", "fork"]
    ),
682
    # Path to the cache for storing downloaded assets
683
    "VLLM_ASSETS_CACHE": lambda: os.path.expanduser(
684
685
686
        os.getenv(
            "VLLM_ASSETS_CACHE",
            os.path.join(get_default_cache_root(), "vllm", "assets"),
687
688
        )
    ),
689
690
    # If the env var is set, we will clean model file in
    # this path $VLLM_ASSETS_CACHE/model_streamer/$model_name
691
692
693
    "VLLM_ASSETS_CACHE_MODEL_CLEAN": lambda: bool(
        int(os.getenv("VLLM_ASSETS_CACHE_MODEL_CLEAN", "0"))
    ),
694
695
    # Timeout for fetching images when serving multimodal models
    # Default is 5 seconds
696
    "VLLM_IMAGE_FETCH_TIMEOUT": lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
697
    # Timeout for fetching videos when serving multimodal models
698
    # Default is 30 seconds
699
700
701
    "VLLM_VIDEO_FETCH_TIMEOUT": lambda: int(
        os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "30")
    ),
702
    # Timeout for fetching audio when serving multimodal models
703
    # Default is 10 seconds
704
705
706
    "VLLM_AUDIO_FETCH_TIMEOUT": lambda: int(
        os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")
    ),
707
708
    # Whether to allow HTTP redirects when fetching from media URLs.
    # Default to True
709
710
711
    "VLLM_MEDIA_URL_ALLOW_REDIRECTS": lambda: bool(
        int(os.getenv("VLLM_MEDIA_URL_ALLOW_REDIRECTS", "1"))
    ),
712
713
714
    # Max number of workers for the thread pool handling
    # media bytes loading. Set to 1 to disable parallel processing.
    # Default is 8
715
716
717
    "VLLM_MEDIA_LOADING_THREAD_COUNT": lambda: int(
        os.getenv("VLLM_MEDIA_LOADING_THREAD_COUNT", "8")
    ),
718
719
720
    # Maximum filesize in MB for a single audio file when processing
    # speech-to-text requests. Files larger than this will be rejected.
    # Default is 25 MB
721
722
723
    "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB": lambda: int(
        os.getenv("VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", "25")
    ),
724
725
726
727
728
729
730
    # Backend for Video IO
    # - "opencv": Default backend that uses OpenCV stream buffered backend.
    #
    # Custom backend implementations can be registered
    # via `@VIDEO_LOADER_REGISTRY.register("my_custom_video_loader")` and
    # imported at runtime.
    # If a non-existing backend is used, an AssertionError will be thrown.
731
732
733
    "VLLM_VIDEO_LOADER_BACKEND": lambda: os.getenv(
        "VLLM_VIDEO_LOADER_BACKEND", "opencv"
    ),
734
    # [DEPRECATED] Cache size (in GiB per process) for multimodal input cache
735
    # Default is 4 GiB per API process + 4 GiB per engine core process
736
    "VLLM_MM_INPUT_CACHE_GIB": lambda: int(os.getenv("VLLM_MM_INPUT_CACHE_GIB", "4")),
737
738
    # Path to the XLA persistent cache directory.
    # Only used for XLA devices such as TPUs.
739
    "VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser(
740
        os.getenv(
741
            "VLLM_XLA_CACHE_PATH",
742
            os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
743
744
        )
    ),
745
    # If set, assert on XLA recompilation after each execution step.
746
747
748
    "VLLM_XLA_CHECK_RECOMPILATION": lambda: bool(
        int(os.getenv("VLLM_XLA_CHECK_RECOMPILATION", "0"))
    ),
749
    # Enable SPMD mode for TPU backend.
750
751
752
753
    "VLLM_XLA_USE_SPMD": lambda: bool(int(os.getenv("VLLM_XLA_USE_SPMD", "0"))),
    "VLLM_FUSED_MOE_CHUNK_SIZE": lambda: int(
        os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "32768")
    ),
754
755
756
    # Control whether to use fused MoE activation chunking. Current chunking
    # logic is incompatible with torch.compile and causes IMA. See issue
    # https://github.com/vllm-project/vllm/issues/19631.
757
758
759
    "VLLM_ENABLE_FUSED_MOE_ACTIVATION_CHUNKING": lambda: bool(
        int(os.getenv("VLLM_ENABLE_FUSED_MOE_ACTIVATION_CHUNKING", "1"))
    ),
760
761
    # If set, the OpenAI API server will stay alive even after the underlying
    # AsyncLLMEngine errors and stops serving requests
762
763
764
    "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH": lambda: bool(
        os.getenv("VLLM_KEEP_ALIVE_ON_ENGINE_DEATH", 0)
    ),
765
766
767
768
    # If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows
    # the user to specify a max sequence length greater than
    # the max length derived from the model's config.json.
    # To enable this, set VLLM_ALLOW_LONG_MAX_MODEL_LEN=1.
769
770
771
772
    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": lambda: (
        os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower()
        in ("1", "true")
    ),
773
774
    # If set, forces FP8 Marlin to be used for FP8 quantization regardless
    # of the hardware support for FP8 compute.
775
776
777
778
779
780
781
    "VLLM_TEST_FORCE_FP8_MARLIN": lambda: (
        os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower()
        in ("1", "true")
    ),
    "VLLM_TEST_FORCE_LOAD_FORMAT": lambda: os.getenv(
        "VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"
    ),
782
783
    # Time in ms for the zmq client to wait for a response from the backend
    # server for simple data operations
784
    "VLLM_RPC_TIMEOUT": lambda: int(os.getenv("VLLM_RPC_TIMEOUT", "10000")),
785
    # Timeout in seconds for keeping HTTP connections alive in API server
786
787
788
    "VLLM_HTTP_TIMEOUT_KEEP_ALIVE": lambda: int(
        os.environ.get("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5")
    ),
789
790
791
    # a list of plugin names to load, separated by commas.
    # if this is not set, it means all plugins will be loaded
    # if this is set to an empty string, no plugins will be loaded
792
793
794
    "VLLM_PLUGINS": lambda: None
    if "VLLM_PLUGINS" not in os.environ
    else os.environ["VLLM_PLUGINS"].split(","),
795
796
797
    # a local directory to look in for unrecognized LoRA adapters.
    # only works if plugins are enabled and
    # VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled.
798
799
800
    "VLLM_LORA_RESOLVER_CACHE_DIR": lambda: os.getenv(
        "VLLM_LORA_RESOLVER_CACHE_DIR", None
    ),
801
802
803
804
    # Enables torch profiler if set.
    # Both AsyncLLM's CPU traces as well as workers'
    # traces (CPU & GPU) will be saved under this directory.
    # Note that it must be an absolute path.
805
806
807
808
809
810
811
    "VLLM_TORCH_PROFILER_DIR": lambda: (
        None
        if os.getenv("VLLM_TORCH_PROFILER_DIR", None) is None
        else os.path.abspath(
            os.path.expanduser(os.getenv("VLLM_TORCH_PROFILER_DIR", "."))
        )
    ),
812
813
814
    # Enable torch profiler to record shapes if set
    # VLLM_TORCH_PROFILER_RECORD_SHAPES=1. If not set, torch profiler will
    # not record shapes.
815
816
817
    "VLLM_TORCH_PROFILER_RECORD_SHAPES": lambda: bool(
        os.getenv("VLLM_TORCH_PROFILER_RECORD_SHAPES", "0") != "0"
    ),
818
819
820
    # Enable torch profiler to profile memory if set
    # VLLM_TORCH_PROFILER_WITH_PROFILE_MEMORY=1. If not set, torch profiler
    # will not profile memory.
821
822
823
    "VLLM_TORCH_PROFILER_WITH_PROFILE_MEMORY": lambda: bool(
        os.getenv("VLLM_TORCH_PROFILER_WITH_PROFILE_MEMORY", "0") != "0"
    ),
824
825
826
    # Enable torch profiler to profile stack if set
    # VLLM_TORCH_PROFILER_WITH_STACK=1. If not set, torch profiler WILL
    # profile stack by default.
827
828
829
    "VLLM_TORCH_PROFILER_WITH_STACK": lambda: bool(
        os.getenv("VLLM_TORCH_PROFILER_WITH_STACK", "1") != "0"
    ),
830
831
832
    # Enable torch profiler to profile flops if set
    # VLLM_TORCH_PROFILER_WITH_FLOPS=1. If not set, torch profiler will
    # not profile flops.
833
834
835
    "VLLM_TORCH_PROFILER_WITH_FLOPS": lambda: bool(
        os.getenv("VLLM_TORCH_PROFILER_WITH_FLOPS", "0") != "0"
    ),
836
    # If set, vLLM will use Triton implementations of AWQ.
837
    "VLLM_USE_TRITON_AWQ": lambda: bool(int(os.getenv("VLLM_USE_TRITON_AWQ", "0"))),
838
    # If set, allow loading or unloading lora adapters in runtime,
839
840
841
842
    "VLLM_ALLOW_RUNTIME_LORA_UPDATING": lambda: (
        os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower()
        in ("1", "true")
    ),
843
844
845
846
847
848
    # We assume drivers can report p2p status correctly.
    # If the program hangs when using custom allreduce,
    # potantially caused by a bug in the driver (535 series),
    # if might be helpful to set VLLM_SKIP_P2P_CHECK=0
    # so that vLLM can verify if p2p is actually working.
    # See https://github.com/vllm-project/vllm/blob/a9b15c606fea67a072416ea0ea115261a2756058/vllm/distributed/device_communicators/custom_all_reduce_utils.py#L101-L108 for details. # noqa
849
    "VLLM_SKIP_P2P_CHECK": lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "1") == "1",
850
851
852
853
    # List of quantization kernels that should be disabled, used for testing
    # and performance comparisons. Currently only affects MPLinearKernel
    # selection
    # (kernels: MacheteLinearKernel, MarlinLinearKernel, ExllamaLinearKernel)
854
855
856
    "VLLM_DISABLED_KERNELS": lambda: []
    if "VLLM_DISABLED_KERNELS" not in os.environ
    else os.environ["VLLM_DISABLED_KERNELS"].split(","),
857
    # Disable pynccl (using torch.distributed instead)
858
859
860
    "VLLM_DISABLE_PYNCCL": lambda: (
        os.getenv("VLLM_DISABLE_PYNCCL", "False").lower() in ("true", "1")
    ),
861
    # If set, use the V1 code path.
862
    "VLLM_USE_V1": lambda: bool(int(os.getenv("VLLM_USE_V1", "1"))),
863
864
    # Disable aiter ops unless specifically enabled.
    # Acts as a parent switch to enable the rest of the other operations.
865
866
867
    "VLLM_ROCM_USE_AITER": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1")
    ),
868
869
    # Whether to use aiter paged attention.
    # By default is disabled.
870
871
872
    "VLLM_ROCM_USE_AITER_PAGED_ATTN": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_PAGED_ATTN", "False").lower() in ("true", "1")
    ),
873
874
875
    # use aiter linear op if aiter ops are enabled
    # The following list of related ops
    # - scaled_mm (per-tensor / rowwise)
876
877
878
    "VLLM_ROCM_USE_AITER_LINEAR": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_LINEAR", "True").lower() in ("true", "1")
    ),
879
880
    # Whether to use aiter moe ops.
    # By default is enabled.
881
882
883
    "VLLM_ROCM_USE_AITER_MOE": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
    ),
884
    # use aiter rms norm op if aiter ops are enabled.
885
886
887
    "VLLM_ROCM_USE_AITER_RMSNORM": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_RMSNORM", "True").lower() in ("true", "1")
    ),
888
889
    # Whether to use aiter mla ops.
    # By default is enabled.
890
891
892
    "VLLM_ROCM_USE_AITER_MLA": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1")
    ),
893
894
    # Whether to use aiter mha ops.
    # By default is enabled.
895
896
897
    "VLLM_ROCM_USE_AITER_MHA": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_MHA", "True").lower() in ("true", "1")
    ),
898
899
    # Whether to use aiter fp4 gemm asm.
    # By default is disabled.
900
901
902
    "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", "False").lower() in ("true", "1")
    ),
903
904
    # Whether to use aiter rope.
    # By default is disabled.
905
906
907
    "VLLM_ROCM_USE_TRITON_ROPE": lambda: (
        os.getenv("VLLM_ROCM_USE_TRITON_ROPE", "False").lower() in ("true", "1")
    ),
908
909
    # Whether to use aiter triton fp8 bmm kernel
    # By default is enabled.
910
911
912
    "VLLM_ROCM_USE_AITER_FP8BMM": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_FP8BMM", "True").lower() in ("true", "1")
    ),
913
914
915
916
917
    # Use AITER triton unified attention for V1 attention
    "VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION", "False").lower()
        in ("true", "1")
    ),
918
919
920
921
922
923
    # Whether to use aiter fusion shared experts ops.
    # By default is enabled.
    "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": lambda: (
        os.getenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "True").lower()
        in ("true", "1")
    ),
924
    # use rocm skinny gemms
925
926
927
    "VLLM_ROCM_USE_SKINNY_GEMM": lambda: (
        os.getenv("VLLM_ROCM_USE_SKINNY_GEMM", "True").lower() in ("true", "1")
    ),
928
    # Pad the fp8 weights to 256 bytes for ROCm
929
    "VLLM_ROCM_FP8_PADDING": lambda: bool(int(os.getenv("VLLM_ROCM_FP8_PADDING", "1"))),
930
    # Pad the weights for the moe kernel
931
    "VLLM_ROCM_MOE_PADDING": lambda: bool(int(os.getenv("VLLM_ROCM_MOE_PADDING", "1"))),
932
    # custom paged attention kernel for MI3* cards
933
934
935
    "VLLM_ROCM_CUSTOM_PAGED_ATTN": lambda: (
        os.getenv("VLLM_ROCM_CUSTOM_PAGED_ATTN", "True").lower() in ("true", "1")
    ),
936
937
938
    # Custom quick allreduce kernel for MI3* cards
    # Choice of quantization level: FP, INT8, INT6, INT4 or NONE
    # Recommended for large models to get allreduce
939
940
941
942
943
    "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": env_with_choices(
        "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION",
        "NONE",
        ["FP", "INT8", "INT6", "INT4", "NONE"],
    ),
944
945
946
947
    # Custom quick allreduce kernel for MI3* cards
    # Due to the lack of the bfloat16 asm instruction, bfloat16
    # kernels are slower than fp16,
    # If environment variable is set to 1, the input is converted to fp16
948
949
950
951
    "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16": lambda: (
        os.getenv("VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", "True").lower()
        in ("true", "1")
    ),
952
953
954
955
956
957
    # Custom quick allreduce kernel for MI3* cards.
    # Controls the maximum allowed number of data bytes(MB) for custom quick
    # allreduce communication.
    # Default: 2048 MB.
    # Data exceeding this size will use either custom allreduce or RCCL
    # communication.
958
959
960
    "VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB": lambda: maybe_convert_int(
        os.environ.get("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", None)
    ),
961
    # Divisor for dynamic query scale factor calculation for FP8 KV Cache
962
    "Q_SCALE_CONSTANT": lambda: int(os.getenv("Q_SCALE_CONSTANT", "200")),
963
    # Divisor for dynamic key scale factor calculation for FP8 KV Cache
964
    "K_SCALE_CONSTANT": lambda: int(os.getenv("K_SCALE_CONSTANT", "200")),
965
    # Divisor for dynamic value scale factor calculation for FP8 KV Cache
966
    "V_SCALE_CONSTANT": lambda: int(os.getenv("V_SCALE_CONSTANT", "100")),
967
    # If set, enable multiprocessing in LLM for the V1 code path.
968
969
970
971
972
973
    "VLLM_ENABLE_V1_MULTIPROCESSING": lambda: bool(
        int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))
    ),
    "VLLM_LOG_BATCHSIZE_INTERVAL": lambda: float(
        os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")
    ),
974
    "VLLM_DISABLE_COMPILE_CACHE": disable_compile_cache,
975
976
977
    # If set, vllm will run in development mode, which will enable
    # some additional endpoints for developing and debugging,
    # e.g. `/reset_prefix_cache`
978
    "VLLM_SERVER_DEV_MODE": lambda: bool(int(os.getenv("VLLM_SERVER_DEV_MODE", "0"))),
979
980
981
982
983
984
985
    # Controls the maximum number of requests to handle in a
    # single asyncio task when processing per-token outputs in the
    # V1 AsyncLLM interface. It is applicable when handling a high
    # concurrency of streaming requests.
    # Setting this too high can result in a higher variance of
    # inter-message latencies. Setting it too low can negatively impact
    # TTFT and overall throughput.
986
987
988
    "VLLM_V1_OUTPUT_PROC_CHUNK_SIZE": lambda: int(
        os.getenv("VLLM_V1_OUTPUT_PROC_CHUNK_SIZE", "128")
    ),
989
    # If set, vLLM will disable the MLA attention optimizations.
990
    "VLLM_MLA_DISABLE": lambda: bool(int(os.getenv("VLLM_MLA_DISABLE", "0"))),
991
992
    # If set, vLLM will pick up the provided Flash Attention MLA
    # max number splits for cuda graph decode
993
994
995
    "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": lambda: int(
        os.getenv("VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH", "32")
    ),
996
997
998
    # Number of GPUs per worker in Ray, if it is set to be a fraction,
    # it allows ray to schedule multiple actors on a single GPU,
    # so that users can colocate other actors on the same GPUs as vLLM.
999
1000
1001
    "VLLM_RAY_PER_WORKER_GPUS": lambda: float(
        os.getenv("VLLM_RAY_PER_WORKER_GPUS", "1.0")
    ),
1002
1003
1004
    # Bundle indices for Ray, if it is set, it can control precisely
    # which indices are used for the Ray bundle, for every worker.
    # Format: comma-separated list of integers, e.g. "0,1,2,3"
1005
    "VLLM_RAY_BUNDLE_INDICES": lambda: os.getenv("VLLM_RAY_BUNDLE_INDICES", ""),
1006
1007
    # In some system, find_loaded_library() may not work. So we allow users to
    # specify the path through environment variable VLLM_CUDART_SO_PATH.
1008
    "VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None),
1009
    # Rank of the process in the data parallel setting
1010
    "VLLM_DP_RANK": lambda: int(os.getenv("VLLM_DP_RANK", "0")),
1011
1012
    # Rank of the process in the data parallel setting.
    # Defaults to VLLM_DP_RANK when not set.
1013
1014
1015
    "VLLM_DP_RANK_LOCAL": lambda: int(
        os.getenv("VLLM_DP_RANK_LOCAL", sys.modules[__name__].VLLM_DP_RANK)
    ),
1016
    # World size of the data parallel setting
1017
    "VLLM_DP_SIZE": lambda: int(os.getenv("VLLM_DP_SIZE", "1")),
1018
    # IP address of the master node in the data parallel setting
1019
    "VLLM_DP_MASTER_IP": lambda: os.getenv("VLLM_DP_MASTER_IP", "127.0.0.1"),
1020
    # Port of the master node in the data parallel setting
1021
    "VLLM_DP_MASTER_PORT": lambda: int(os.getenv("VLLM_DP_MASTER_PORT", "0")),
1022
1023
1024
1025
1026
    # In the context of executing MoE models with Data-Parallel, Expert-Parallel
    # and Batched All-to-All dispatch/combine kernels, VLLM_MOE_DP_CHUNK_SIZE
    # dictates the quantum of tokens that can be dispatched from a DP
    # rank. All DP ranks process the activations in VLLM_MOE_DP_CHUNK_SIZE
    # units.
1027
    "VLLM_MOE_DP_CHUNK_SIZE": lambda: int(os.getenv("VLLM_MOE_DP_CHUNK_SIZE", "256")),
1028
    # Randomize inputs during dummy runs when using Data Parallel
1029
1030
1031
1032
    "VLLM_RANDOMIZE_DP_DUMMY_INPUTS": lambda: os.environ.get(
        "VLLM_RANDOMIZE_DP_DUMMY_INPUTS", "0"
    )
    == "1",
1033
1034
1035
1036
1037
1038
1039
    # Strategy to pack the data parallel ranks for Ray.
    # Available options:
    # - "fill":
    #   for DP master node, allocate exactly data-parallel-size-local DP ranks,
    #   for non-master nodes, allocate as many DP ranks as can fit;
    # - "strict":
    #   allocate exactly data-parallel-size-local DP ranks to each picked node;
1040
1041
1042
    # - "span":
    #   Should be used only when a single DP rank requires multiple nodes.
    #   allocate one DP rank over as many nodes as required for set world_size;
1043
1044
1045
1046
    # This environment variable is ignored if data-parallel-backend is not Ray.
    "VLLM_RAY_DP_PACK_STRATEGY": lambda: os.getenv(
        "VLLM_RAY_DP_PACK_STRATEGY", "strict"
    ),
1047
    # Whether to use S3 path for model loading in CI via RunAI Streamer
1048
    "VLLM_CI_USE_S3": lambda: os.environ.get("VLLM_CI_USE_S3", "0") == "1",
1049
    # Use model_redirect to redirect the model name to a local folder.
1050
1051
1052
1053
1054
    # `model_redirect` can be a json file mapping the model between
    # repo_id and local folder:
    # {"meta-llama/Llama-3.2-1B": "/tmp/Llama-3.2-1B"}
    # or a space separated values table file:
    # meta-llama/Llama-3.2-1B   /tmp/Llama-3.2-1B
1055
1056
1057
    "VLLM_MODEL_REDIRECT_PATH": lambda: os.environ.get(
        "VLLM_MODEL_REDIRECT_PATH", None
    ),
1058
    # Whether to use atomicAdd reduce in gptq/awq marlin kernel.
1059
1060
1061
1062
    "VLLM_MARLIN_USE_ATOMIC_ADD": lambda: os.environ.get(
        "VLLM_MARLIN_USE_ATOMIC_ADD", "0"
    )
    == "1",
1063
    # Whether to use marlin kernel in mxfp4 quantization method
1064
1065
1066
    "VLLM_MXFP4_USE_MARLIN": lambda: maybe_convert_bool(
        os.environ.get("VLLM_MXFP4_USE_MARLIN", None)
    ),
1067
1068
1069
    # Whether to turn on the outlines cache for V1
    # This cache is unbounded and on disk, so it's not safe to use in
    # an environment with potentially malicious users.
1070
1071
1072
1073
    "VLLM_V1_USE_OUTLINES_CACHE": lambda: os.environ.get(
        "VLLM_V1_USE_OUTLINES_CACHE", "0"
    )
    == "1",
1074
1075
    # Gap between padding buckets for the forward pass. So we have
    # 8, we will run forward pass with [16, 24, 32, ...].
1076
1077
1078
1079
1080
1081
1082
1083
    "VLLM_TPU_BUCKET_PADDING_GAP": lambda: int(
        os.environ["VLLM_TPU_BUCKET_PADDING_GAP"]
    )
    if "VLLM_TPU_BUCKET_PADDING_GAP" in os.environ
    else 0,
    "VLLM_TPU_MOST_MODEL_LEN": lambda: maybe_convert_int(
        os.environ.get("VLLM_TPU_MOST_MODEL_LEN", None)
    ),
1084
    # Whether using Pathways
1085
1086
1087
    "VLLM_TPU_USING_PATHWAYS": lambda: bool(
        "proxy" in os.getenv("JAX_PLATFORMS", "").lower()
    ),
1088
    # Allow use of DeepGemm kernels for fused moe ops.
1089
    "VLLM_USE_DEEP_GEMM": lambda: bool(int(os.getenv("VLLM_USE_DEEP_GEMM", "1"))),
1090
    # Whether to use E8M0 scaling when DeepGEMM is used on Blackwell GPUs.
1091
1092
1093
    "VLLM_USE_DEEP_GEMM_E8M0": lambda: bool(
        int(os.getenv("VLLM_USE_DEEP_GEMM_E8M0", "1"))
    ),
1094
1095
1096
1097
    # DeepGemm JITs the kernels on-demand. The warmup attempts to make DeepGemm
    # JIT all the required kernels before model execution so there is no
    # JIT'ing in the hot-path. However, this warmup increases the engine
    # startup time by a couple of minutes.
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
    # Available options:
    #  - "skip"  : Skip warmup.
    #  - "full"  : Warmup deepgemm by running all possible gemm shapes the
    #   engine could encounter.
    #  - "relax" : Select gemm shapes to run based on some heuristics. The
    #   heuristic aims to have the same effect as running all possible gemm
    #   shapes, but provides no guarantees.
    "VLLM_DEEP_GEMM_WARMUP": env_with_choices(
        "VLLM_DEEP_GEMM_WARMUP",
        "relax",
        [
            "skip",
            "full",
            "relax",
        ],
1113
    ),
1114
    # Whether to use fused grouped_topk used for MoE expert selection.
1115
1116
1117
    "VLLM_USE_FUSED_MOE_GROUPED_TOPK": lambda: bool(
        int(os.getenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "1"))
    ),
1118
    # Allow use of FlashInfer MoE kernels for fused moe ops.
1119
1120
1121
    "VLLM_USE_FLASHINFER_MOE_FP16": lambda: bool(
        int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP16", "0"))
    ),
1122
    # Allow use of FlashInfer MoE kernels for fused moe ops.
1123
1124
1125
    "VLLM_USE_FLASHINFER_MOE_FP8": lambda: bool(
        int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP8", "0"))
    ),
1126
    # Allow use of FlashInfer CUTLASS kernels for fused moe ops.
1127
1128
1129
    "VLLM_USE_FLASHINFER_MOE_FP4": lambda: bool(
        int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP4", "0"))
    ),
1130
1131
    # If set to 1, use the FlashInfer
    # MXFP8 (activation) x MXFP4 (weight) MoE backend.
1132
1133
1134
    "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": lambda: bool(
        int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "0"))
    ),
1135
1136
1137
1138
    # If set to 1, use the FlashInfer CUTLASS backend for
    # MXFP8 (activation) x MXFP4 (weight) MoE.
    # This is separate from the TRTLLMGEN path controlled by
    # VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8.
1139
1140
1141
    "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS": lambda: bool(
        int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "0"))
    ),
1142
1143
    # If set to 1, use the FlashInfer
    # BF16 (activation) x MXFP4 (weight) MoE backend.
1144
1145
1146
    "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16": lambda: bool(
        int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "0"))
    ),
1147
1148
1149
    # Control the cache sized used by the xgrammar compiler. The default
    # of 512 MB should be enough for roughly 1000 JSON schemas.
    # It can be changed with this variable if needed for some reason.
1150
    "VLLM_XGRAMMAR_CACHE_MB": lambda: int(os.getenv("VLLM_XGRAMMAR_CACHE_MB", "512")),
1151
1152
1153
1154
1155
1156
1157
    # Control the threshold for msgspec to use 'zero copy' for
    # serialization/deserialization of tensors. Tensors below
    # this limit will be encoded into the msgpack buffer, and
    # tensors above will instead be sent via a separate message.
    # While the sending side still actually copies the tensor
    # in all cases, on the receiving side, tensors above this
    # limit will actually be zero-copy decoded.
1158
1159
1160
    "VLLM_MSGPACK_ZERO_COPY_THRESHOLD": lambda: int(
        os.getenv("VLLM_MSGPACK_ZERO_COPY_THRESHOLD", "256")
    ),
1161
1162
1163
    # If set, allow insecure serialization using pickle.
    # This is useful for environments where it is deemed safe to use the
    # insecure method and it is needed for some reason.
1164
1165
1166
    "VLLM_ALLOW_INSECURE_SERIALIZATION": lambda: bool(
        int(os.getenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0"))
    ),
Robert Shaw's avatar
Robert Shaw committed
1167
    # IP address used for NIXL handshake between remote agents.
1168
1169
1170
    "VLLM_NIXL_SIDE_CHANNEL_HOST": lambda: os.getenv(
        "VLLM_NIXL_SIDE_CHANNEL_HOST", "localhost"
    ),
Robert Shaw's avatar
Robert Shaw committed
1171
    # Port used for NIXL handshake between remote agents.
1172
1173
1174
    "VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
        os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")
    ),
1175
    # all2all backend for vllm's expert parallel communication
1176
    # Available options:
1177
1178
1179
    # - "naive": naive all2all implementation using broadcasts
    # - "allgather_reducescatter": all2all implementation based on allgather and
    #  reducescatter
1180
    # - "pplx": use pplx kernels
1181
1182
    # - "deepep_high_throughput", use deepep high-throughput kernels
    # - "deepep_low_latency", use deepep low-latency kernels
1183
    # - "flashinfer_all2allv", use flashinfer alltoallv kernels for mnnvl
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
    "VLLM_ALL2ALL_BACKEND": env_with_choices(
        "VLLM_ALL2ALL_BACKEND",
        "allgather_reducescatter",
        [
            "naive",
            "pplx",
            "deepep_high_throughput",
            "deepep_low_latency",
            "allgather_reducescatter",
            "flashinfer_all2allv",
        ],
    ),
1196
1197
    # Flashinfer MoE backend for vLLM's fused Mixture-of-Experts support.
    # Both require compute capability 10.0 or above.
1198
1199
1200
1201
1202
    # Available options:
    # - "throughput":  [default]
    #     Uses CUTLASS kernels optimized for high-throughput batch inference.
    # - "latency":
    #     Uses TensorRT-LLM kernels optimized for low-latency inference.
1203
1204
1205
    "VLLM_FLASHINFER_MOE_BACKEND": env_with_choices(
        "VLLM_FLASHINFER_MOE_BACKEND", "throughput", ["throughput", "latency"]
    ),
1206
1207
1208
1209
    # Control the maximum number of tokens per expert supported by the
    # NVFP4 MoE CUTLASS Kernel. This value is used to create a buffer for
    # the blockscale tensor of activations NVFP4 Quantization.
    # This is used to prevent the kernel from running out of memory.
1210
1211
1212
    "VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE": lambda: int(
        os.getenv("VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE", "163840")
    ),
1213
1214
1215
1216
    # Specifies the thresholds of the communicated tensor sizes under which
    # vllm should use flashinfer fused allreduce. The variable should be a
    # JSON with the following format:
    #     { <world size>: <max size in mb> }
1217
    # Unspecified world sizes will fall back to
1218
    #     { 2: 64, 4: 1, <everything else>: 0.5 }
1219
1220
1221
    "VLLM_FLASHINFER_ALLREDUCE_FUSION_THRESHOLDS_MB": lambda: json.loads(
        os.getenv("VLLM_FLASHINFER_ALLREDUCE_FUSION_THRESHOLDS_MB", "{}")
    ),
1222
1223
1224
1225
1226
1227
    # MoE routing strategy selector.
    # See `RoutingSimulator.get_available_strategies()` # for available
    # strategies.
    # Cutstom routing strategies can be registered by
    # RoutingSimulator.register_strategy()
    # Note: custom strategies may not produce correct model outputs
1228
1229
1230
    "VLLM_MOE_ROUTING_SIMULATION_STRATEGY": lambda: os.environ.get(
        "VLLM_MOE_ROUTING_SIMULATION_STRATEGY", ""
    ).lower(),
1231
    # Regex timeout for use by the vLLM tool parsing plugins.
1232
1233
1234
    "VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
        os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
    ),
1235
1236
    # Reduce CPU usage when vLLM is idle. Enabling this will incur small
    # latency penalty when a request eventually comes.
1237
    "VLLM_SLEEP_WHEN_IDLE": lambda: bool(int(os.getenv("VLLM_SLEEP_WHEN_IDLE", "0"))),
1238
1239
1240
    # Control the max chunk bytes (in MB) for the rpc message queue.
    # Object larger than this threshold will be broadcast to worker
    # processes via zmq.
1241
1242
1243
    "VLLM_MQ_MAX_CHUNK_BYTES_MB": lambda: int(
        os.getenv("VLLM_MQ_MAX_CHUNK_BYTES_MB", "16")
    ),
1244
1245
    # Timeout in seconds for execute_model RPC calls in multiprocessing
    # executor (only applies when TP > 1).
1246
1247
1248
    "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": lambda: int(
        os.getenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "300")
    ),
1249
1250
1251
1252
1253
1254
1255
    # KV Cache layout used throughout vllm.
    # Some common values are:
    # - NHD
    # - HND
    # Where N=num_blocks, H=num_heads and D=head_size. The default value will
    # leave the layout choice to the backend. Mind that backends may only
    # implement and support a subset of all possible layouts.
1256
1257
1258
    "VLLM_KV_CACHE_LAYOUT": env_with_choices(
        "VLLM_KV_CACHE_LAYOUT", None, ["NHD", "HND"]
    ),
1259
1260
1261
    # Enable checking whether the generated logits contain NaNs,
    # indicating corrupted output. Useful for debugging low level bugs
    # or bad hardware but it may add compute overhead.
1262
1263
1264
    "VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
        int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
    ),
1265
1266
1267
    # Controls whether or not emulations are used for NVFP4
    # generations on machines < 100 for compressed-tensors
    # models
1268
1269
1270
    "VLLM_USE_NVFP4_CT_EMULATIONS": lambda: bool(
        int(os.getenv("VLLM_USE_NVFP4_CT_EMULATIONS", "0"))
    ),
1271
1272
1273
1274
    # Time (in seconds) after which the KV cache on the producer side is
    # automatically cleared if no READ notification is received from the
    # consumer. This is only applicable when using NixlConnector in a
    # disaggregated decode-prefill setup.
1275
1276
1277
    "VLLM_NIXL_ABORT_REQUEST_TIMEOUT": lambda: int(
        os.getenv("VLLM_NIXL_ABORT_REQUEST_TIMEOUT", "480")
    ),
1278
    # Controls whether or not to use cudnn prefill
1279
1280
1281
    "VLLM_USE_CUDNN_PREFILL": lambda: bool(
        int(os.getenv("VLLM_USE_CUDNN_PREFILL", "0"))
    ),
1282
1283
1284
1285
    # Controls whether to use TRT-LLM ragged DeepSeek prefill
    "VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL": lambda: bool(
        int(os.getenv("VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL", "0"))
    ),
1286
1287
1288
    # If set to 1/True, use the TRTLLM attention backend in flashinfer.
    # If set to 0/False, use the default attention backend in flashinfer.
    # If not set, auto-detect the attention backend in flashinfer.
1289
1290
1291
1292
1293
    "VLLM_USE_TRTLLM_ATTENTION": lambda: (
        None
        if "VLLM_USE_TRTLLM_ATTENTION" not in os.environ
        else os.environ["VLLM_USE_TRTLLM_ATTENTION"].lower() in ("1", "true")
    ),
1294
    # If set to 1, when we use fp8 kv, we do not quantize Q to fp8
1295
1296
1297
    "VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION": lambda: bool(
        int(os.getenv("VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION", "0"))
    ),
1298
1299
    # If set, it means we pre-downloaded cubin files and flashinfer will
    # read the cubin files directly.
1300
    "VLLM_HAS_FLASHINFER_CUBIN": lambda: os.getenv("VLLM_HAS_FLASHINFER_CUBIN", False),
1301
1302
1303
1304
1305
1306
1307
1308
1309
    # Supported options:
    # - "flashinfer-cudnn": use flashinfer cudnn GEMM backend
    # - "flashinfer-trtllm": use flashinfer trtllm GEMM backend
    # - "flashinfer-cutlass": use flashinfer cutlass GEMM backend
    # - <none>: automatically pick an available backend
    "VLLM_NVFP4_GEMM_BACKEND": env_with_choices(
        "VLLM_NVFP4_GEMM_BACKEND",
        None,
        ["flashinfer-cudnn", "flashinfer-trtllm", "flashinfer-cutlass"],
1310
    ),
1311
1312
1313
    # Controls garbage collection during CUDA graph capture.
    # If set to 0 (default), enables GC freezing to speed up capture time.
    # If set to 1, allows GC to run during capture.
1314
1315
1316
    "VLLM_ENABLE_CUDAGRAPH_GC": lambda: bool(
        int(os.getenv("VLLM_ENABLE_CUDAGRAPH_GC", "0"))
    ),
1317
    # Used to force set up loopback IP
1318
    "VLLM_LOOPBACK_IP": lambda: os.getenv("VLLM_LOOPBACK_IP", ""),
1319
1320
1321
    # Used to set the process name prefix for vLLM processes.
    # This is useful for debugging and monitoring purposes.
    # The default value is "VLLM".
1322
    "VLLM_PROCESS_NAME_PREFIX": lambda: os.getenv("VLLM_PROCESS_NAME_PREFIX", "VLLM"),
1323
1324
1325
1326
1327
1328
1329
    # Allow chunked local attention with hybrid kv cache manager.
    # Currently using the Hybrid KV cache manager with chunked local attention
    # in the Llama4 models (the only models currently using chunked local attn)
    # causes a latency regression. For this reason, we disable it by default.
    # This flag is used to allow users to enable it if they want to (to save on
    # kv-cache memory usage and enable longer contexts)
    # TODO(lucas): Remove this flag once latency regression is resolved.
1330
1331
1332
    "VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE": lambda: bool(
        int(os.getenv("VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE", "0"))
    ),
1333
1334
    # Enables support for the "store" option in the OpenAI Responses API.
    # When set to 1, vLLM's OpenAI server will retain the input and output
1335
1336
    # messages for those requests in memory. By default, this is disabled (0),
    # and the "store" option is ignored.
1337
1338
1339
1340
1341
    # NOTE/WARNING:
    # 1. Messages are kept in memory only (not persisted to disk) and will be
    #    lost when the vLLM server shuts down.
    # 2. Enabling this option will cause a memory leak, as stored messages are
    #    never removed from memory until the server terminates.
1342
1343
1344
    "VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
        int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
    ),
xiao-llm's avatar
xiao-llm committed
1345
    # If set, use the fp8 mfma in rocm paged attention.
1346
1347
1348
    "VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
        int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
    ),
1349
    # Whether to use pytorch symmetric memory for allreduce
1350
    "VLLM_ALLREDUCE_USE_SYMM_MEM": lambda: bool(
1351
        int(os.getenv("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"))
1352
    ),
1353
    # Allows vllm to find tuned config under customized folder
1354
    "VLLM_TUNED_CONFIG_FOLDER": lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None),
1355
1356
1357
1358
1359
1360
1361
1362
1363
    # Valid values are container,code_interpreter,web_search_preview
    # ex VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS=container,code_interpreter
    # If the server_label of your mcp tool is not in this list it will
    # be completely ignored.
    "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": env_set_with_choices(
        "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS",
        default=[],
        choices=["container", "code_interpreter", "web_search_preview"],
    ),
1364
    # Allows harmony instructions to be injected on system messages
1365
1366
1367
    "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": lambda: bool(
        int(os.getenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "0"))
    ),
1368
1369
1370
1371
1372
1373
    # Enable automatic retry when tool call JSON parsing fails
    # If enabled, returns an error message to the model to retry
    # If disabled (default), raises an exception and fails the request
    "VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY": lambda: bool(
        int(os.getenv("VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY", "0"))
    ),
1374
    # Add optional custom scopes for profiling, disable to avoid overheads
1375
1376
1377
    "VLLM_CUSTOM_SCOPES_FOR_PROFILING": lambda: bool(
        int(os.getenv("VLLM_CUSTOM_SCOPES_FOR_PROFILING", "0"))
    ),
1378
    # Add optional nvtx scopes for profiling, disable to avoid overheads
1379
1380
1381
    "VLLM_NVTX_SCOPES_FOR_PROFILING": lambda: bool(
        int(os.getenv("VLLM_NVTX_SCOPES_FOR_PROFILING", "0"))
    ),
1382
1383
    # Represent block hashes in KV cache events as 64-bit integers instead of
    # raw bytes. Defaults to True for backward compatibility.
1384
1385
1386
    "VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES": lambda: bool(
        int(os.getenv("VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES", "1"))
    ),
1387
1388
    # Name of the shared memory buffer used for object storage.
    # Only effective when mm_config.mm_processor_cache_type == "shm".
1389
1390
1391
    "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME": lambda: os.getenv(
        "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", "VLLM_OBJECT_STORAGE_SHM_BUFFER"
    ),
1392
    # The size in MB of the buffers (NVL and RDMA) used by DeepEP
1393
1394
1395
    "VLLM_DEEPEP_BUFFER_SIZE_MB": lambda: int(
        os.getenv("VLLM_DEEPEP_BUFFER_SIZE_MB", "1024")
    ),
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
    # Force DeepEP to use intranode kernel for inter-node communication in
    # high throughput mode. This is useful archive higher prefill throuhgput
    # on system supports multi-node nvlink (e.g GB200).
    "VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE": lambda: bool(
        int(os.getenv("VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE", "0"))
    ),
    # Allow DeepEP to use MNNVL (multi-node nvlink) for internode_ll kernel,
    # turn this for better latency on GB200 like system
    "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool(
        int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0"))
    ),
1407
1408
    # The number of SMs to allocate for communication kernels when running DBO
    # the rest of the SMs on the device will be allocated to compute
1409
    "VLLM_DBO_COMM_SMS": lambda: int(os.getenv("VLLM_DBO_COMM_SMS", "20")),
1410
1411
1412
    # Enable max_autotune & coordinate_descent_tuning in inductor_config
    # to compile static shapes passed from compile_sizes in compilation_config
    # If set to 1, enable max_autotune; By default, this is enabled (1)
1413
1414
1415
    "VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE": lambda: bool(
        int(os.getenv("VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE", "1"))
    ),
1416
1417
    # If set to 1, enable coordinate_descent_tuning;
    # By default, this is enabled (1)
1418
1419
1420
    "VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING": lambda: bool(
        int(os.getenv("VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING", "1"))
    ),
1421
    # Flag to enable NCCL symmetric memory allocation and registration
1422
1423
1424
    "VLLM_USE_NCCL_SYMM_MEM": lambda: bool(
        int(os.getenv("VLLM_USE_NCCL_SYMM_MEM", "0"))
    ),
1425
    # NCCL header path
1426
    "VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None),
1427
1428
    # Flag to enable FBGemm kernels on model execution
    "VLLM_USE_FBGEMM": lambda: bool(int(os.getenv("VLLM_USE_FBGEMM", "0"))),
1429
1430
1431
1432
1433
1434
    # GC debug config
    # - VLLM_GC_DEBUG=0: disable GC debugger
    # - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times
    # - VLLM_GC_DEBUG='{"top_objects":5}': enable GC debugger with
    #                                      top 5 collected objects
    "VLLM_GC_DEBUG": lambda: os.getenv("VLLM_GC_DEBUG", ""),
1435
1436
1437
1438
    # Disables parallel execution of shared_experts via separate cuda stream
    "VLLM_DISABLE_SHARED_EXPERTS_STREAM": lambda: os.getenv(
        "VLLM_DISABLE_SHARED_EXPERTS_STREAM", False
    ),
1439
1440
}

1441
# --8<-- [end:env-vars-definition]
1442

1443

1444
def __getattr__(name: str):
1445
1446
1447
1448
1449
1450
    """
    Gets environment variables lazily.

    NOTE: After enable_envs_cache() invocation (which triggered after service
    initialization), all environment variables will be cached.
    """
1451
1452
1453
1454
1455
    if name in environment_variables:
        return environment_variables[name]()
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")


1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
def enable_envs_cache() -> None:
    """
    Enables caching of environment variables. This is useful for performance
    reasons, as it avoids the need to re-evaluate environment variables on
    every call.

    NOTE: Currently, it's invoked after service initialization to reduce
    runtime overhead. This also means that environment variables should NOT
    be updated after the service is initialized.
    """
    # Tag __getattr__ with functools.cache
    global __getattr__
    __getattr__ = functools.cache(__getattr__)

    # Cache all environment variables
    for key in environment_variables:
        __getattr__(key)


1475
1476
def __dir__():
    return list(environment_variables.keys())
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490


def is_set(name: str):
    """Check if an environment variable is explicitly set."""
    if name in environment_variables:
        return name in os.environ
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")


def set_vllm_use_v1(use_v1: bool):
    if is_set("VLLM_USE_V1"):
        raise ValueError(
            "Should not call set_vllm_use_v1() if VLLM_USE_V1 is set "
            "explicitly by the user. Please raise this as a Github "
1491
1492
            "Issue and explicitly set VLLM_USE_V1=0 or 1."
        )
1493
    os.environ["VLLM_USE_V1"] = "1" if use_v1 else "0"
1494
1495
1496
1497
1498
1499
1500
1501


def compute_hash() -> str:
    """
    WARNING: Whenever a new key is added to this environment
    variables, ensure that it is included in the factors list if
    it affects the computation graph. For example, different values
    of VLLM_PP_LAYER_PARTITION will generate different computation
1502
    graphs, so it is included in the factors list. The env vars that
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
    affect the choice of different kernels or attention backends should
    also be included in the factors list.
    """

    # The values of envs may affects the computation graph.
    # TODO(DefTruth): hash all environment variables?
    # for key in environment_variables:
    #     factorize(key)
    environment_variables_to_hash = [
        "VLLM_PP_LAYER_PARTITION",
        "VLLM_MLA_DISABLE",
1514
        "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH",
1515
1516
1517
1518
        "VLLM_USE_TRITON_FLASH_ATTN",
        "VLLM_USE_TRITON_AWQ",
        "VLLM_DP_RANK",
        "VLLM_DP_SIZE",
1519
        "VLLM_USE_STANDALONE_COMPILE",
1520
        "VLLM_FUSED_MOE_CHUNK_SIZE",
1521
1522
1523
1524
1525
1526
        "VLLM_FLASHINFER_MOE_BACKEND",
        "VLLM_V1_USE_PREFILL_DECODE_ATTENTION",
        "VLLM_ATTENTION_BACKEND",
        "VLLM_USE_FLASHINFER_SAMPLER",
        "VLLM_DISABLED_KERNELS",
        "VLLM_USE_DEEP_GEMM",
1527
        "VLLM_USE_DEEP_GEMM_E8M0",
1528
        "VLLM_USE_FUSED_MOE_GROUPED_TOPK",
1529
        "VLLM_USE_FLASHINFER_MOE_FP16",
1530
1531
1532
        "VLLM_USE_FLASHINFER_MOE_FP8",
        "VLLM_USE_FLASHINFER_MOE_FP4",
        "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8",
1533
        "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS",
1534
1535
        "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16",
        "VLLM_USE_CUDNN_PREFILL",
1536
        "VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL",
1537
        "VLLM_USE_TRTLLM_ATTENTION",
1538
        "VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION",
1539
1540
1541
1542
1543
1544
1545
        "VLLM_ROCM_USE_AITER",
        "VLLM_ROCM_USE_AITER_PAGED_ATTN",
        "VLLM_ROCM_USE_AITER_LINEAR",
        "VLLM_ROCM_USE_AITER_MOE",
        "VLLM_ROCM_USE_AITER_RMSNORM",
        "VLLM_ROCM_USE_AITER_MLA",
        "VLLM_ROCM_USE_AITER_MHA",
1546
1547
        "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
        "VLLM_ROCM_USE_TRITON_ROPE",
1548
        "VLLM_ROCM_USE_AITER_FP8BMM",
1549
        "VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION",
1550
1551
1552
1553
1554
1555
1556
        "VLLM_ROCM_USE_SKINNY_GEMM",
        "VLLM_ROCM_FP8_PADDING",
        "VLLM_ROCM_MOE_PADDING",
        "VLLM_ROCM_CUSTOM_PAGED_ATTN",
        "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION",
        "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16",
        "VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB",
xiao-llm's avatar
xiao-llm committed
1557
        "VLLM_ROCM_FP8_MFMA_PAGE_ATTN",
1558
1559
        "VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE",
        "VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING",
1560
        "VLLM_NVFP4_GEMM_BACKEND",
1561
        "VLLM_USE_FBGEMM",
1562
1563
        "VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE",
        "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL",
1564
1565
    ]
    for key in environment_variables_to_hash:
1566
1567
        # if this goes out of sync with environment_variables,
        # it's not a user error, it's a bug
1568
        assert key in environment_variables, (
1569
            "Please update environment_variables_to_hash in envs.py"
1570
        )
1571

1572
    factors = [environment_variables[key]() for key in environment_variables_to_hash]
1573

1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
    ray_noset_env_vars = [
        # Refer to
        # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/nvidia_gpu.py#L11
        # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/amd_gpu.py#L11
        # https://github.com/ray-project/ray/blob/b97d21dab233c2bd8ed7db749a82a1e594222b5c/python/ray/_private/accelerators/amd_gpu.py#L10
        # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/npu.py#L12
        # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/hpu.py#L12
        # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/neuron.py#L14
        # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/tpu.py#L38
        # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/intel_gpu.py#L10
        # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/rbln.py#L10
        "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES",
        "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES",
        "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES",
        "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES",
        "RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES",
        "RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES",
        "RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS",
        "RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR",
        "RAY_EXPERIMENTAL_NOSET_RBLN_RT_VISIBLE_DEVICES",
    ]
    factors.extend([os.getenv(var) for var in ray_noset_env_vars])

1597
    hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
1598
1599

    return hash_str