---
title: "VLLMDeployModelParameters Configuration"
description: "Local deploy model parameters."
---

import { ConfigDetail } from "@site/src/components/mdx/ConfigDetail";

<ConfigDetail config={{
  "name": "VLLMDeployModelParameters",
  "description": "Local deploy model parameters.",
  "documentationUrl": "",
  "parameters": [
    {
      "name": "name",
      "type": "string",
      "required": true,
      "description": "The name of the model."
    },
    {
      "name": "path",
      "type": "string",
      "required": false,
      "description": "The path of the model, if you want to deploy a local model."
    },
    {
      "name": "backend",
      "type": "string",
      "required": false,
      "description": "The real model name to pass to the provider, default is None. If backend is None, use name as the real model name."
    },
    {
      "name": "device",
      "type": "string",
      "required": false,
      "description": "Device to run model. If None, the device is automatically determined",
      "defaultValue": "auto"
    },
    {
      "name": "provider",
      "type": "string",
      "required": false,
      "description": "The provider of the model. If model is deployed in local, this is the inference type. If model is deployed in third-party service, this is platform name('proxy/<platform>')",
      "defaultValue": "vllm"
    },
    {
      "name": "verbose",
      "type": "boolean",
      "required": false,
      "description": "Show verbose output.",
      "defaultValue": "False"
    },
    {
      "name": "concurrency",
      "type": "integer",
      "required": false,
      "description": "Model concurrency limit",
      "defaultValue": "100"
    },
    {
      "name": "prompt_template",
      "type": "string",
      "required": false,
      "description": "Prompt template. If None, the prompt template is automatically determined from model. Just for local deployment."
    },
    {
      "name": "context_length",
      "type": "integer",
      "required": false,
      "description": "The context length of the model. If None, it is automatically determined from model."
    },
    {
      "name": "reasoning_model",
      "type": "boolean",
      "required": false,
      "description": "Whether the model is a reasoning model. If None, it is automatically determined from model."
    },
    {
      "name": "trust_remote_code",
      "type": "boolean",
      "required": false,
      "description": "Trust remote code or not.",
      "defaultValue": "True"
    },
    {
      "name": "download_dir",
      "type": "string",
      "required": false,
      "description": "Directory to download and load the weights, default to the default cache dir of huggingface."
    },
    {
      "name": "load_format",
      "type": "string",
      "required": false,
      "description": "The format of the model weights to load.\n\n* \"auto\" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available.\n* \"pt\" will load the weights in the pytorch bin format.\n* \"safetensors\" will load the weights in the safetensors format.\n* \"npcache\" will load the weights in pytorch format and store a numpy cache to speed up the loading.\n* \"dummy\" will initialize the weights with random values, which is mainly for profiling.\n* \"tensorizer\" will load the weights using tensorizer from CoreWeave. See the Tensorize vLLM Model script in the Examples section for more information.\n* \"runai_streamer\" will load the Safetensors weights using Run:aiModel Streamer \n* \"bitsandbytes\" will load the weights using bitsandbytes quantization.\n",
      "defaultValue": "auto",
      "validValues": [
        "auto",
        "pt",
        "safetensors",
        "npcache",
        "dummy",
        "tensorizer",
        "runai_streamer",
        "bitsandbytes",
        "sharded_state",
        "gguf",
        "mistral"
      ]
    },
    {
      "name": "config_format",
      "type": "string",
      "required": false,
      "description": "The format of the model config to load.\n\n* \"auto\" will try to load the config in hf format if available else it will try to load in mistral format ",
      "defaultValue": "auto",
      "validValues": [
        "auto",
        "hf",
        "mistral"
      ]
    },
    {
      "name": "dtype",
      "type": "string",
      "required": false,
      "description": "Data type for model weights and activations.\n\n* \"auto\" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models.\n* \"half\" for FP16. Recommended for AWQ quantization.\n* \"float16\" is the same as \"half\".\n* \"bfloat16\" for a balance between precision and range.\n* \"float\" is shorthand for FP32 precision.\n* \"float32\" for FP32 precision.",
      "defaultValue": "auto",
      "validValues": [
        "auto",
        "half",
        "float16",
        "bfloat16",
        "float",
        "float32"
      ]
    },
    {
      "name": "kv_cache_dtype",
      "type": "string",
      "required": false,
      "description": "Data type for kv cache storage. If \"auto\", will use model data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports fp8 (=fp8_e4m3)",
      "defaultValue": "auto",
      "validValues": [
        "auto",
        "fp8",
        "fp8_e5m2",
        "fp8_e4m3"
      ]
    },
    {
      "name": "seed",
      "type": "integer",
      "required": false,
      "description": "Random seed for operations.",
      "defaultValue": "0"
    },
    {
      "name": "max_model_len",
      "type": "integer",
      "required": false,
      "description": "Model context length. If unspecified, will be automatically derived from the model config."
    },
    {
      "name": "distributed_executor_backend",
      "type": "string",
      "required": false,
      "description": "Backend to use for distributed model workers, either \"ray\" or \"mp\" (multiprocessing). If the product of pipeline_parallel_size and tensor_parallel_size is less than or equal to the number of GPUs available, \"mp\" will be used to keep processing on a single host. Otherwise, this will default to \"ray\" if Ray is installed and fail otherwise. Note that tpu only supports Ray for distributed inference.",
      "validValues": [
        "ray",
        "mp",
        "uni",
        "external_launcher"
      ]
    },
    {
      "name": "pipeline_parallel_size",
      "type": "integer",
      "required": false,
      "description": "Number of pipeline stages.",
      "defaultValue": "1"
    },
    {
      "name": "tensor_parallel_size",
      "type": "integer",
      "required": false,
      "description": "Number of tensor parallel replicas.",
      "defaultValue": "1"
    },
    {
      "name": "max_parallel_loading_workers",
      "type": "integer",
      "required": false,
      "description": "Load model sequentially in multiple batches, to avoid RAM OOM when using tensor parallel and large models."
    },
    {
      "name": "block_size",
      "type": "integer",
      "required": false,
      "description": "Token block size for contiguous chunks of tokens. This is ignored on neuron devices and set to ``--max-model-len``. On CUDA devices, only block sizes up to 32 are supported. On HPU devices, block size defaults to 128.",
      "validValues": [
        "8",
        "16",
        "32",
        "64",
        "128"
      ]
    },
    {
      "name": "enable_prefix_caching",
      "type": "boolean",
      "required": false,
      "description": "Enables automatic prefix caching. "
    },
    {
      "name": "swap_space",
      "type": "number",
      "required": false,
      "description": "CPU swap space size (GiB) per GPU.",
      "defaultValue": "4"
    },
    {
      "name": "cpu_offload_gb",
      "type": "number",
      "required": false,
      "description": "The space in GiB to offload to CPU, per GPU. Default is 0, which means no offloading. Intuitively, this argument can be seen as a virtual way to increase the GPU memory size. For example, if you have one 24 GB GPU and set this to 10, virtually you can think of it as a 34 GB GPU. Then you can load a 13B model with BF16 weight, which requires at least 26GB GPU memory. Note that this requires fast CPU-GPU interconnect, as part of the model is loaded from CPU memory to GPU memory on the fly in each model forward pass.",
      "defaultValue": "0"
    },
    {
      "name": "gpu_memory_utilization",
      "type": "number",
      "required": false,
      "description": "The fraction of GPU memory to be used for the model executor, which can range from 0 to 1. For example, a value of 0.5 would imply 50%% GPU memory utilization. If unspecified, will use the default value of 0.9. This is a per-instance limit, and only applies to the current vLLM instance.It does not matter if you have another vLLM instance running on the same GPU. For example, if you have two vLLM instances running on the same GPU, you can set the GPU memory utilization to 0.5 for each instance.",
      "defaultValue": "0.9"
    },
    {
      "name": "max_num_batched_tokens",
      "type": "integer",
      "required": false,
      "description": "Maximum number of batched tokens per iteration."
    },
    {
      "name": "max_num_seqs",
      "type": "integer",
      "required": false,
      "description": "Maximum number of sequences per iteration."
    },
    {
      "name": "max_logprobs",
      "type": "integer",
      "required": false,
      "description": "Max number of log probs to return logprobs is specified in SamplingParams.",
      "defaultValue": "20"
    },
    {
      "name": "revision",
      "type": "string",
      "required": false,
      "description": "The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version."
    },
    {
      "name": "code_revision",
      "type": "string",
      "required": false,
      "description": "The specific revision to use for the model code on Hugging Face Hub. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version."
    },
    {
      "name": "tokenizer_revision",
      "type": "string",
      "required": false,
      "description": "Revision of the huggingface tokenizer to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version."
    },
    {
      "name": "tokenizer_mode",
      "type": "string",
      "required": false,
      "description": "The tokenizer mode.\n\n* \"auto\" will use the fast tokenizer if available.\n* \"slow\" will always use the slow tokenizer. \n* \"mistral\" will always use the `mistral_common` tokenizer.",
      "defaultValue": "auto",
      "validValues": [
        "auto",
        "slow",
        "mistral"
      ]
    },
    {
      "name": "quantization",
      "type": "string",
      "required": false,
      "description": "Method used to quantize the weights. If None, we first check the `quantization_config` attribute in the model config file. If that is None, we assume the model weights are not quantized and use `dtype` to determine the data type of the weights.",
      "validValues": [
        "aqlm",
        "awq",
        "deepspeedfp",
        "tpu_int8",
        "fp8",
        "ptpc_fp8",
        "fbgemm_fp8",
        "modelopt",
        "marlin",
        "gguf",
        "gptq_marlin_24",
        "gptq_marlin",
        "awq_marlin",
        "gptq",
        "compressed-tensors",
        "bitsandbytes",
        "qqq",
        "hqq",
        "experts_int8",
        "neuron_quant",
        "ipex",
        "quark",
        "moe_wna16"
      ]
    },
    {
      "name": "max_seq_len_to_capture",
      "type": "integer",
      "required": false,
      "description": "Maximum sequence length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode. Additionally for encoder-decoder models, if the sequence length of the encoder input is larger than this, we fall back to the eager mode.",
      "defaultValue": "8192"
    },
    {
      "name": "worker_cls",
      "type": "string",
      "required": false,
      "description": "The worker class to use for distributed execution.",
      "defaultValue": "auto"
    },
    {
      "name": "extras",
      "type": "object",
      "required": false,
      "description": "Extra parameters, it will be passed to the vllm engine."
    }
  ]
}} />

