# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
backend: pytorch

# WideEP related settings
moe_config:
  backend: WIDEEP
  # moe_max_num_tokens will default to max_num_tokens if left unspecified.
  #
  # If you want to set this value explicitly, one recommendation is below:
  #   moe_max_num_tokens = max_batch_size * moe_expert_parallel_size
  #   4096 = 256 * 16
  # moe_max_num_tokens: 4096
  load_balancer: /mnt/engine_configs/deepseek_r1/wide_ep/eplb.yaml

tensor_parallel_size: 16
moe_expert_parallel_size: 16

enable_attention_dp: true
max_batch_size: 256
max_num_tokens: 256
max_seq_len: 8448

kv_cache_config:
  free_gpu_memory_fraction: 0.3
  dtype: fp8

cuda_graph_config:
  enable_padding: true
  batch_sizes:
  - 1
  - 2
  - 4
  - 8
  - 16
  - 32
  - 64
  - 128
  - 256