// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 use std::collections::HashMap; use serde::{de::DeserializeOwned, Deserialize, Serialize}; #[derive(Debug, Default, Clone, Serialize, Deserialize, Eq, PartialEq)] pub struct ModelRuntimeConfig { pub total_kv_blocks: Option, pub max_num_seqs: Option, pub max_num_batched_tokens: Option, pub tool_call_parser: Option, pub reasoning_parser: Option, /// Mapping of engine-specific runtime configs #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub runtime_data: HashMap, } impl ModelRuntimeConfig { pub fn new() -> Self { Self::default() } pub fn set_engine_specific(&mut self, key: &str, value: T) -> anyhow::Result<()> { self.runtime_data .insert(key.to_string(), serde_json::to_value(value)?); Ok(()) } pub fn get_engine_specific(&self, key: &str) -> anyhow::Result> { if let Some(value) = self.runtime_data.get(key) { Ok(Some(serde_json::from_value(value.clone())?)) } else { Ok(None) } } }