# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
backend: pytorch

# WideEP related settings
moe_backend: WideEP
# moe_max_num_tokens will default to max_num_tokens if left unspecified.
#
# If you want to set this value explicitly, one recommendation is below:
#   moe_max_num_tokens = max_batch_size * moe_expert_parallel_size
#   4096 = 256 * 16
# moe_max_num_tokens: 4096
moe_load_balancer: /mnt/engine_configs/eplb.yaml
tensor_parallel_size: 16
moe_expert_parallel_size: 16

enable_attention_dp: true
max_batch_size: 256
max_num_tokens: 256
max_seq_len: 8448
kv_cache_config:
  free_gpu_memory_fraction: 0.7
use_cuda_graph: true
cuda_graph_padding_enabled: true
cuda_graph_batch_sizes:
- 1
- 2
- 4
- 8
- 16
- 32
- 64
- 128
- 256
kv_cache_dtype: fp8
