# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
backend: pytorch
tensor_parallel_size: 16
moe_expert_parallel_size: 16
enable_attention_dp: true
max_batch_size: 256
max_num_tokens: 256
max_seq_len: 8448
kv_cache_config:
  free_gpu_memory_fraction: 0.8
use_cuda_graph: true
cuda_graph_padding_enabled: true
cuda_graph_batch_sizes:
- 1
- 2
- 4
- 8
- 16
- 32
- 64
- 128
- 256
kv_cache_dtype: fp8
