# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: v1
kind: ConfigMap
metadata:
  name: llm-config-specdec
data:
  config.yaml: |
    backend: pytorch
    trust_remote_code: true
    allreduce_strategy: MNNVL
    max_num_tokens: 148000
    enable_chunked_prefill: false
    enable_attention_dp: false
    max_batch_size: 16
    cuda_graph_config:
      batch_sizes:
        - 1
        - 2
        - 4
        - 8
        - 16
      enable_padding: true
    kv_cache_config:
      dtype: fp8
      enable_block_reuse: true
      free_gpu_memory_fraction: 0.75
    moe_config:
      backend: TRTLLM
      max_num_tokens: 2048
    moe_expert_parallel_size: 8
    num_postprocess_workers: 8
    print_iter_log: true
    stream_interval: 10
    tensor_parallel_size: 8
    speculative_config:
      decoding_type: Eagle
      max_draft_len: 3
      speculative_model_dir: /opt/models/hub/models--nvidia--Kimi-K2.5-Thinking-Eagle3/snapshots/0b0c6ac039089ad2c2418c91c039553381a302d9
      allow_advanced_sampling: true
---
apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: kimi-k25-agg-specdec
spec:
  backendFramework: trtllm
  pvcs:
    - name: model-cache
      create: false
  services:
    Frontend:
      componentType: frontend
      replicas: 1
      extraPodSpec:
        mainContainer:
          command:
            - /bin/sh
            - -c
          args:
            - python3 -m dynamo.frontend --router-mode kv --router-reset-states --http-port 8000 --discovery-backend kubernetes --request-plane nats
          image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:my-tag
    TrtllmWorker:
      componentType: worker
      envFromSecret: hf-token-secret
      replicas: 4
      multinode:
        nodeCount: 2
      volumeMounts:
        - name: model-cache
          mountPoint: /opt/models
      sharedMemory:
        size: 179Gi
      resources:
        limits:
          gpu: "4"
        requests:
          gpu: "4"
      extraPodSpec:
        mainContainer:
          command:
            - /bin/sh
            - -c
          args:
            - |
              python3 -m dynamo.trtllm \
                --model-path "${MODEL_NAME}" \
                --served-model-name "${SERVED_MODEL_NAME}" \
                --extra-engine-args "${ENGINE_ARGS}" \
                --discovery-backend kubernetes \
                --request-plane nats \
                --publish-events-and-metrics \
                --kv-block-size 32 \
                --dyn-tool-call-parser kimi_k2 \
                --dyn-reasoning-parser kimi_k25
          image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:my-tag
          env:
            - name: MODEL_NAME
              value: nvidia/Kimi-K2.5-NVFP4
            - name: SERVED_MODEL_NAME
              value: nvidia/Kimi-K2.5-NVFP4
            - name: ENGINE_ARGS
              value: /opt/dynamo/configs/config.yaml
            - name: HF_HOME
              value: /opt/models
            - name: NCCL_DEBUG
              value: INFO
            - name: TRTLLM_ENABLE_PDL
              value: "1"
            - name: ENABLE_CONFIGURABLE_MOE
              value: "1"
            - name: TLLM_LOG_LEVEL
              value: INFO
          volumeMounts:
            - name: llm-config-specdec
              mountPath: /opt/dynamo/configs
              readOnly: true
          workingDir: /workspace/
        volumes:
          - name: llm-config-specdec
            configMap:
              name: llm-config-specdec
