chat_completions.rs 11.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

16
17
18
19
use dynamo_runtime::protocols::annotated::AnnotationsProvider;
use serde::{Deserialize, Serialize};
use validator::Validate;

20
21
22
use crate::engines::ValidateRequest;

use super::{
23
    OpenAIOutputOptionsProvider, OpenAISamplingOptionsProvider, OpenAIStopConditionsProvider,
24
25
26
    common_ext::{
        CommonExt, CommonExtProvider, choose_with_deprecation, emit_nvext_deprecation_warning,
    },
27
28
    nvext::NvExt,
    nvext::NvExtProvider,
29
    validate,
30
};
31

32
pub mod aggregator;
33
34
mod delta;

Paul Hendricks's avatar
Paul Hendricks committed
35
pub use aggregator::DeltaAggregator;
36
37
pub use delta::DeltaGenerator;

38
/// A request structure for creating a chat completion, extending OpenAI's
39
/// `CreateChatCompletionRequest` with [`NvExt`] extensions and common fields.
40
41
42
///
/// # Fields
/// - `inner`: The base OpenAI chat completion request, embedded using `serde(flatten)`.
43
44
45
/// - `common`: Common extension fields (ignore_eos, min_tokens) at root level, embedded using `serde(flatten)`.
/// - `nvext`: The optional NVIDIA extension field. See [`NvExt`] for more details.
///   Note: If ignore_eos is specified in both common and nvext, the common (root-level) value takes precedence.
Paul Hendricks's avatar
Paul Hendricks committed
46
#[derive(Serialize, Deserialize, Validate, Debug, Clone)]
47
pub struct NvCreateChatCompletionRequest {
Paul Hendricks's avatar
Paul Hendricks committed
48
    #[serde(flatten)]
49
    pub inner: dynamo_async_openai::types::CreateChatCompletionRequest,
50

51
52
53
    #[serde(flatten, default)]
    pub common: CommonExt,

54
    #[serde(skip_serializing_if = "Option::is_none")]
55
56
57
    pub nvext: Option<NvExt>,
}

58
59
60
61
62
63
/// A response structure for unary chat completion responses, embedding OpenAI's
/// `CreateChatCompletionResponse`.
///
/// # Fields
/// - `inner`: The base OpenAI unary chat completion response, embedded
///   using `serde(flatten)`.
64
pub type NvCreateChatCompletionResponse = dynamo_async_openai::types::CreateChatCompletionResponse;
65

66
67
68
69
70
71
/// A response structure for streamed chat completions, embedding OpenAI's
/// `CreateChatCompletionStreamResponse`.
///
/// # Fields
/// - `inner`: The base OpenAI streaming chat completion response, embedded
///   using `serde(flatten)`.
72
73
pub type NvCreateChatCompletionStreamResponse =
    dynamo_async_openai::types::CreateChatCompletionStreamResponse;
74

75
76
/// Implements `NvExtProvider` for `NvCreateChatCompletionRequest`,
/// providing access to NVIDIA-specific extensions.
77
impl NvExtProvider for NvCreateChatCompletionRequest {
78
    /// Returns a reference to the optional `NvExt` extension, if available.
79
80
81
82
    fn nvext(&self) -> Option<&NvExt> {
        self.nvext.as_ref()
    }

83
    /// Returns `None`, as raw prompt extraction is not implemented.
84
85
86
87
88
    fn raw_prompt(&self) -> Option<String> {
        None
    }
}

89
90
/// Implements `AnnotationsProvider` for `NvCreateChatCompletionRequest`,
/// enabling retrieval and management of request annotations.
91
impl AnnotationsProvider for NvCreateChatCompletionRequest {
92
    /// Retrieves the list of annotations from `NvExt`, if present.
Biswa Panda's avatar
Biswa Panda committed
93
94
95
96
97
98
    fn annotations(&self) -> Option<Vec<String>> {
        self.nvext
            .as_ref()
            .and_then(|nvext| nvext.annotations.clone())
    }

99
100
101
102
103
104
105
    /// Checks whether a specific annotation exists in the request.
    ///
    /// # Arguments
    /// * `annotation` - A string slice representing the annotation to check.
    ///
    /// # Returns
    /// `true` if the annotation exists, `false` otherwise.
Biswa Panda's avatar
Biswa Panda committed
106
107
108
109
110
111
112
113
    fn has_annotation(&self, annotation: &str) -> bool {
        self.nvext
            .as_ref()
            .and_then(|nvext| nvext.annotations.as_ref())
            .map(|annotations| annotations.contains(&annotation.to_string()))
            .unwrap_or(false)
    }
}
114

115
116
/// Implements `OpenAISamplingOptionsProvider` for `NvCreateChatCompletionRequest`,
/// exposing OpenAI's sampling parameters for chat completion.
117
impl OpenAISamplingOptionsProvider for NvCreateChatCompletionRequest {
118
    /// Retrieves the temperature parameter for sampling, if set.
119
    fn get_temperature(&self) -> Option<f32> {
Paul Hendricks's avatar
Paul Hendricks committed
120
        self.inner.temperature
121
122
    }

123
    /// Retrieves the top-p (nucleus sampling) parameter, if set.
124
    fn get_top_p(&self) -> Option<f32> {
Paul Hendricks's avatar
Paul Hendricks committed
125
        self.inner.top_p
126
127
    }

128
    /// Retrieves the frequency penalty parameter, if set.
129
    fn get_frequency_penalty(&self) -> Option<f32> {
Paul Hendricks's avatar
Paul Hendricks committed
130
        self.inner.frequency_penalty
131
132
    }

133
    /// Retrieves the presence penalty parameter, if set.
134
    fn get_presence_penalty(&self) -> Option<f32> {
Paul Hendricks's avatar
Paul Hendricks committed
135
        self.inner.presence_penalty
136
137
    }

138
    /// Returns a reference to the optional `NvExt` extension, if available.
139
140
141
142
143
    fn nvext(&self) -> Option<&NvExt> {
        self.nvext.as_ref()
    }
}

144
145
146
147
148
149
150
151
152
153
/// Implements `CommonExtProvider` for `NvCreateChatCompletionRequest`,
/// providing access to common extension fields.
impl CommonExtProvider for NvCreateChatCompletionRequest {
    /// Returns a reference to the CommonExt struct.
    fn common_ext(&self) -> Option<&CommonExt> {
        Some(&self.common)
    }

    /// Guided Decoding Options
    fn get_guided_json(&self) -> Option<&serde_json::Value> {
154
155
156
157
158
159
        // Note: This one needs special handling since it returns a reference
        if let Some(nvext) = &self.nvext
            && nvext.guided_json.is_some()
        {
            emit_nvext_deprecation_warning("guided_json", true, self.common.guided_json.is_some());
        }
160
161
162
163
164
165
166
        self.common
            .guided_json
            .as_ref()
            .or_else(|| self.nvext.as_ref().and_then(|nv| nv.guided_json.as_ref()))
    }

    fn get_guided_regex(&self) -> Option<String> {
167
168
169
170
171
        choose_with_deprecation(
            "guided_regex",
            self.common.guided_regex.as_ref(),
            self.nvext.as_ref().and_then(|nv| nv.guided_regex.as_ref()),
        )
172
173
174
    }

    fn get_guided_grammar(&self) -> Option<String> {
175
176
177
178
179
180
181
        choose_with_deprecation(
            "guided_grammar",
            self.common.guided_grammar.as_ref(),
            self.nvext
                .as_ref()
                .and_then(|nv| nv.guided_grammar.as_ref()),
        )
182
183
184
    }

    fn get_guided_choice(&self) -> Option<Vec<String>> {
185
186
187
188
189
        choose_with_deprecation(
            "guided_choice",
            self.common.guided_choice.as_ref(),
            self.nvext.as_ref().and_then(|nv| nv.guided_choice.as_ref()),
        )
190
191
192
    }

    fn get_guided_decoding_backend(&self) -> Option<String> {
193
194
195
        choose_with_deprecation(
            "guided_decoding_backend",
            self.common.guided_decoding_backend.as_ref(),
196
197
            self.nvext
                .as_ref()
198
199
                .and_then(|nv| nv.guided_decoding_backend.as_ref()),
        )
200
    }
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218

    fn get_top_k(&self) -> Option<i32> {
        choose_with_deprecation(
            "top_k",
            self.common.top_k.as_ref(),
            self.nvext.as_ref().and_then(|nv| nv.top_k.as_ref()),
        )
    }

    fn get_repetition_penalty(&self) -> Option<f32> {
        choose_with_deprecation(
            "repetition_penalty",
            self.common.repetition_penalty.as_ref(),
            self.nvext
                .as_ref()
                .and_then(|nv| nv.repetition_penalty.as_ref()),
        )
    }
219
220
221
222

    fn get_include_stop_str_in_output(&self) -> Option<bool> {
        self.common.include_stop_str_in_output
    }
223
224
}

225
226
/// Implements `OpenAIStopConditionsProvider` for `NvCreateChatCompletionRequest`,
/// providing access to stop conditions that control chat completion behavior.
227
impl OpenAIStopConditionsProvider for NvCreateChatCompletionRequest {
228
    /// Retrieves the maximum number of tokens allowed in the response.
229
    #[allow(deprecated)]
Paul Hendricks's avatar
Paul Hendricks committed
230
    fn get_max_tokens(&self) -> Option<u32> {
231
        self.inner.max_completion_tokens.or(self.inner.max_tokens)
232
233
    }

234
    /// Retrieves the minimum number of tokens required in the response.
235
236
    /// Returns `min_tokens` Value
    /// `min_tokens` is not an OpenAI-supported parameter.
Paul Hendricks's avatar
Paul Hendricks committed
237
    fn get_min_tokens(&self) -> Option<u32> {
238
        self.common.min_tokens
239
240
    }

241
242
243
244
245
246
247
    /// Retrieves the stop conditions that terminate the chat completion response.
    ///
    /// Converts OpenAI's `Stop` enum to a `Vec<String>`, normalizing the representation.
    ///
    /// # Returns
    /// * `Some(Vec<String>)` if stop conditions are set.
    /// * `None` if no stop conditions are defined.
248
    fn get_stop(&self) -> Option<Vec<String>> {
249
        self.inner.stop.as_ref().map(|stop| match stop {
250
251
            dynamo_async_openai::types::Stop::String(s) => vec![s.clone()],
            dynamo_async_openai::types::Stop::StringArray(arr) => arr.clone(),
252
        })
253
254
    }

255
    /// Returns a reference to the optional `NvExt` extension, if available.
256
257
258
    fn nvext(&self) -> Option<&NvExt> {
        self.nvext.as_ref()
    }
259
260
261
262
263

    /// Get ignore_eos from CommonExt.
    fn get_common_ignore_eos(&self) -> Option<bool> {
        self.common.ignore_eos
    }
264
265
266
267
268
269
270
271
272
273

    /// Get the effective ignore_eos value, considering both CommonExt and NvExt.
    /// CommonExt (root-level) takes precedence over NvExt.
    fn get_ignore_eos(&self) -> Option<bool> {
        choose_with_deprecation(
            "ignore_eos",
            self.get_common_ignore_eos().as_ref(),
            NvExtProvider::nvext(self).and_then(|nv| nv.ignore_eos.as_ref()),
        )
    }
274
}
275

Greg Clark's avatar
Greg Clark committed
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
impl OpenAIOutputOptionsProvider for NvCreateChatCompletionRequest {
    fn get_logprobs(&self) -> Option<u32> {
        match self.inner.logprobs {
            Some(true) => match self.inner.top_logprobs {
                Some(top_logprobs) => Some(top_logprobs as u32),
                None => Some(1_u32),
            },
            Some(false) => None,
            None => None,
        }
    }

    fn get_prompt_logprobs(&self) -> Option<u32> {
        None
    }

    fn get_skip_special_tokens(&self) -> Option<bool> {
        None
    }

    fn get_formatted_prompt(&self) -> Option<bool> {
        None
    }
}

301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
/// Implements `ValidateRequest` for `NvCreateChatCompletionRequest`,
/// allowing us to validate the data.
impl ValidateRequest for NvCreateChatCompletionRequest {
    fn validate(&self) -> Result<(), anyhow::Error> {
        validate::validate_messages(&self.inner.messages)?;
        validate::validate_model(&self.inner.model)?;
        // none for store
        validate::validate_reasoning_effort(&self.inner.reasoning_effort)?;
        validate::validate_metadata(&self.inner.metadata)?;
        validate::validate_frequency_penalty(self.inner.frequency_penalty)?;
        validate::validate_logit_bias(&self.inner.logit_bias)?;
        // none for logprobs
        validate::validate_top_logprobs(self.inner.top_logprobs)?;
        // validate::validate_max_tokens(self.inner.max_tokens)?; // warning depricated field
        validate::validate_max_completion_tokens(self.inner.max_completion_tokens)?;
        validate::validate_n(self.inner.n)?;
        // none for modalities
        // none for prediction
        // none for audio
        validate::validate_presence_penalty(self.inner.presence_penalty)?;
        // none for response_format
        // none for seed
        validate::validate_service_tier(&self.inner.service_tier)?;
        validate::validate_stop(&self.inner.stop)?;
        // none for stream
        // none for stream_options
        validate::validate_temperature(self.inner.temperature)?;
        validate::validate_top_p(self.inner.top_p)?;
        validate::validate_tools(&self.inner.tools.as_deref())?;
        // none for tool_choice
        // none for parallel_tool_calls
        validate::validate_user(self.inner.user.as_deref())?;
        // none for function call
        // none for functions

        Ok(())
    }
}