completions.rs 22 KB
Newer Older
1
// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
4
// SPDX-License-Identifier: Apache-2.0

use derive_builder::Builder;
5
use dynamo_runtime::protocols::annotated::AnnotationsProvider;
6
use serde::{Deserialize, Serialize};
7
use utoipa::ToSchema;
8
9
use validator::Validate;

10
11
use crate::engines::ValidateRequest;

12
use super::{
13
14
    ContentProvider, OpenAIOutputOptionsProvider, OpenAISamplingOptionsProvider,
    OpenAIStopConditionsProvider,
Greg Clark's avatar
Greg Clark committed
15
    common::{self, OutputOptionsProvider, SamplingOptionsProvider, StopConditionsProvider},
16
    common_ext::{CommonExt, CommonExtProvider},
17
    nvext::{NvExt, NvExtProvider},
18
    validate,
19
20
};

21
22
23
24
25
mod aggregator;
mod delta;

pub use aggregator::DeltaAggregator;
pub use delta::DeltaGenerator;
Biswa Panda's avatar
Biswa Panda committed
26

27
#[derive(ToSchema, Serialize, Deserialize, Validate, Debug, Clone)]
28
pub struct NvCreateCompletionRequest {
29
    #[serde(flatten)]
30
    #[schema(value_type = Object)]
31
    pub inner: dynamo_protocols::types::CreateCompletionRequest,
32

33
34
35
    #[serde(flatten)]
    pub common: CommonExt,

36
37
    #[serde(skip_serializing_if = "Option::is_none")]
    pub nvext: Option<NvExt>,
38
39
40
41

    // metadata - passthrough parameter without restrictions
    #[serde(skip_serializing_if = "Option::is_none")]
    pub metadata: Option<serde_json::Value>,
42
43
44
45

    /// Catch-all for unsupported fields - checked during validation
    #[serde(flatten, default, skip_serializing)]
    pub unsupported_fields: std::collections::HashMap<String, serde_json::Value>,
46
47
}

48
#[derive(ToSchema, Serialize, Deserialize, Validate, Debug, Clone)]
49
50
pub struct NvCreateCompletionResponse {
    #[serde(flatten)]
51
    #[schema(value_type = Object)]
52
    pub inner: dynamo_protocols::types::CreateCompletionResponse,
53
54
    #[serde(skip_serializing_if = "Option::is_none")]
    pub nvext: Option<serde_json::Value>,
55
56
}

57
impl ContentProvider for dynamo_protocols::types::Choice {
58
59
60
61
62
    fn content(&self) -> String {
        self.text.clone()
    }
}

63
pub fn prompt_to_string(prompt: &dynamo_protocols::types::Prompt) -> String {
64
    match prompt {
65
66
67
        dynamo_protocols::types::Prompt::String(s) => s.clone(),
        dynamo_protocols::types::Prompt::StringArray(arr) => arr.join(" "), // Join strings with spaces
        dynamo_protocols::types::Prompt::IntegerArray(arr) => arr
68
69
70
71
            .iter()
            .map(|&num| num.to_string())
            .collect::<Vec<_>>()
            .join(" "),
72
        dynamo_protocols::types::Prompt::ArrayOfIntegerArray(arr) => arr
73
74
75
76
77
78
79
80
81
82
83
84
85
            .iter()
            .map(|inner| {
                inner
                    .iter()
                    .map(|&num| num.to_string())
                    .collect::<Vec<_>>()
                    .join(" ")
            })
            .collect::<Vec<_>>()
            .join(" | "), // Separate arrays with a delimiter
    }
}

86
/// Get the batch size from a prompt (1 for single prompts, array length for batch prompts)
87
pub fn get_prompt_batch_size(prompt: &dynamo_protocols::types::Prompt) -> usize {
88
    match prompt {
89
90
91
92
        dynamo_protocols::types::Prompt::String(_) => 1,
        dynamo_protocols::types::Prompt::IntegerArray(_) => 1,
        dynamo_protocols::types::Prompt::StringArray(arr) => arr.len(),
        dynamo_protocols::types::Prompt::ArrayOfIntegerArray(arr) => arr.len(),
93
94
95
96
97
98
99
    }
}

/// Extract a single prompt from a batch at the given index.
/// For single prompts, returns a clone regardless of index.
/// For batch prompts, returns the prompt at the specified index.
pub fn extract_single_prompt(
100
    prompt: &dynamo_protocols::types::Prompt,
101
    index: usize,
102
) -> dynamo_protocols::types::Prompt {
103
    match prompt {
104
105
        dynamo_protocols::types::Prompt::String(s) => {
            dynamo_protocols::types::Prompt::String(s.clone())
106
        }
107
108
        dynamo_protocols::types::Prompt::IntegerArray(arr) => {
            dynamo_protocols::types::Prompt::IntegerArray(arr.clone())
109
        }
110
111
        dynamo_protocols::types::Prompt::StringArray(arr) => {
            dynamo_protocols::types::Prompt::String(arr[index].clone())
112
        }
113
114
        dynamo_protocols::types::Prompt::ArrayOfIntegerArray(arr) => {
            dynamo_protocols::types::Prompt::IntegerArray(arr[index].clone())
115
116
117
118
        }
    }
}

119
impl NvExtProvider for NvCreateCompletionRequest {
120
121
122
123
124
    fn nvext(&self) -> Option<&NvExt> {
        self.nvext.as_ref()
    }

    fn raw_prompt(&self) -> Option<String> {
125
126
127
128
129
        if let Some(nvext) = self.nvext.as_ref()
            && let Some(use_raw_prompt) = nvext.use_raw_prompt
            && use_raw_prompt
        {
            return Some(prompt_to_string(&self.inner.prompt));
130
131
132
133
134
        }
        None
    }
}

135
impl AnnotationsProvider for NvCreateCompletionRequest {
Biswa Panda's avatar
Biswa Panda committed
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    fn annotations(&self) -> Option<Vec<String>> {
        self.nvext
            .as_ref()
            .and_then(|nvext| nvext.annotations.clone())
    }

    fn has_annotation(&self, annotation: &str) -> bool {
        self.nvext
            .as_ref()
            .and_then(|nvext| nvext.annotations.as_ref())
            .map(|annotations| annotations.contains(&annotation.to_string()))
            .unwrap_or(false)
    }
}
150

151
impl OpenAISamplingOptionsProvider for NvCreateCompletionRequest {
152
    fn get_temperature(&self) -> Option<f32> {
153
        self.inner.temperature
154
155
156
    }

    fn get_top_p(&self) -> Option<f32> {
157
        self.inner.top_p
158
159
160
    }

    fn get_frequency_penalty(&self) -> Option<f32> {
161
        self.inner.frequency_penalty
162
163
164
    }

    fn get_presence_penalty(&self) -> Option<f32> {
165
        self.inner.presence_penalty
166
167
168
169
170
    }

    fn nvext(&self) -> Option<&NvExt> {
        self.nvext.as_ref()
    }
171
172
173
174
175
176
177
178
179
180
181
182

    fn get_seed(&self) -> Option<i64> {
        self.inner.seed
    }

    fn get_n(&self) -> Option<u8> {
        self.inner.n
    }

    fn get_best_of(&self) -> Option<u8> {
        self.inner.best_of
    }
183
184
}

185
186
187
188
189
190
impl CommonExtProvider for NvCreateCompletionRequest {
    fn common_ext(&self) -> Option<&CommonExt> {
        Some(&self.common)
    }

    /// Guided Decoding Options
191
192
    fn get_guided_json(&self) -> Option<serde_json::Value> {
        self.common.guided_json.clone()
193
194
195
    }

    fn get_guided_regex(&self) -> Option<String> {
196
        self.common.guided_regex.clone()
197
198
199
    }

    fn get_guided_grammar(&self) -> Option<String> {
200
        self.common.guided_grammar.clone()
201
202
203
    }

    fn get_guided_choice(&self) -> Option<Vec<String>> {
204
        self.common.guided_choice.clone()
205
206
207
    }

    fn get_guided_decoding_backend(&self) -> Option<String> {
208
        self.common.guided_decoding_backend.clone()
209
    }
210

211
    fn get_guided_whitespace_pattern(&self) -> Option<String> {
212
        self.common.guided_whitespace_pattern.clone()
213
214
    }

215
    fn get_top_k(&self) -> Option<i32> {
216
        self.common.top_k
217
218
    }

219
    fn get_min_p(&self) -> Option<f32> {
220
        self.common.min_p
221
222
    }

223
    fn get_repetition_penalty(&self) -> Option<f32> {
224
        self.common.repetition_penalty
225
    }
226
227
228
229

    fn get_include_stop_str_in_output(&self) -> Option<bool> {
        self.common.include_stop_str_in_output
    }
230
231
232
233

    fn get_skip_special_tokens(&self) -> Option<bool> {
        self.common.skip_special_tokens
    }
234
235
}

236
impl OpenAIStopConditionsProvider for NvCreateCompletionRequest {
Paul Hendricks's avatar
Paul Hendricks committed
237
    fn get_max_tokens(&self) -> Option<u32> {
238
        self.inner.max_tokens
239
240
    }

Paul Hendricks's avatar
Paul Hendricks committed
241
    fn get_min_tokens(&self) -> Option<u32> {
242
        self.common.min_tokens
243
244
245
    }

    fn get_stop(&self) -> Option<Vec<String>> {
246
        use dynamo_protocols::types::Stop;
247
248
249
250
        self.inner.stop.as_ref().map(|s| match s {
            Stop::String(s) => vec![s.clone()],
            Stop::StringArray(arr) => arr.clone(),
        })
251
252
253
254
255
    }

    fn nvext(&self) -> Option<&NvExt> {
        self.nvext.as_ref()
    }
256
257
258
259

    fn get_common_ignore_eos(&self) -> Option<bool> {
        self.common.ignore_eos
    }
260

261
    /// Get the effective ignore_eos value from CommonExt.
262
    fn get_ignore_eos(&self) -> Option<bool> {
263
        self.common.ignore_eos
264
    }
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
}

#[derive(Builder)]
pub struct ResponseFactory {
    #[builder(setter(into))]
    pub model: String,

    #[builder(default)]
    pub system_fingerprint: Option<String>,

    #[builder(default = "format!(\"cmpl-{}\", uuid::Uuid::new_v4())")]
    pub id: String,

    #[builder(default = "\"text_completion\".to_string()")]
    pub object: String,

281
282
    #[builder(default = "chrono::Utc::now().timestamp() as u32")]
    pub created: u32,
283
284
285
286
287
288
289
290
291
}

impl ResponseFactory {
    pub fn builder() -> ResponseFactoryBuilder {
        ResponseFactoryBuilder::default()
    }

    pub fn make_response(
        &self,
292
293
        choice: dynamo_protocols::types::Choice,
        usage: Option<dynamo_protocols::types::CompletionUsage>,
294
    ) -> NvCreateCompletionResponse {
295
        let inner = dynamo_protocols::types::CreateCompletionResponse {
296
297
            id: self.id.clone(),
            object: self.object.clone(),
298
            created: self.created,
299
300
301
302
            model: self.model.clone(),
            choices: vec![choice],
            system_fingerprint: self.system_fingerprint.clone(),
            usage,
303
        };
304
        NvCreateCompletionResponse { inner, nvext: None }
305
306
307
308
    }
}

/// Implements TryFrom for converting an OpenAI's CompletionRequest to an Engine's CompletionRequest
309
impl TryFrom<NvCreateCompletionRequest> for common::CompletionRequest {
310
311
    type Error = anyhow::Error;

312
    fn try_from(request: NvCreateCompletionRequest) -> Result<Self, Self::Error> {
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
        // openai_api_rs::v1::completion::CompletionRequest {
        // NA  pub model: String,
        //     pub prompt: String,
        // **  pub suffix: Option<String>,
        //     pub max_tokens: Option<i32>,
        //     pub temperature: Option<f32>,
        //     pub top_p: Option<f32>,
        //     pub n: Option<i32>,
        //     pub stream: Option<bool>,
        //     pub logprobs: Option<i32>,
        //     pub echo: Option<bool>,
        //     pub stop: Option<Vec<String, Global>>,
        //     pub presence_penalty: Option<f32>,
        //     pub frequency_penalty: Option<f32>,
        //     pub best_of: Option<i32>,
        //     pub logit_bias: Option<HashMap<String, i32, RandomState>>,
        //     pub user: Option<String>,
        // }
        //
        // ** no supported

334
        if request.inner.suffix.is_some() {
335
336
337
338
339
340
341
342
343
344
345
            return Err(anyhow::anyhow!("suffix is not supported"));
        }

        let stop_conditions = request
            .extract_stop_conditions()
            .map_err(|e| anyhow::anyhow!("Failed to extract stop conditions: {}", e))?;

        let sampling_options = request
            .extract_sampling_options()
            .map_err(|e| anyhow::anyhow!("Failed to extract sampling options: {}", e))?;

Greg Clark's avatar
Greg Clark committed
346
347
348
349
        let output_options = request
            .extract_output_options()
            .map_err(|e| anyhow::anyhow!("Failed to extract output options: {}", e))?;

350
        let prompt = common::PromptType::Completion(common::CompletionContext {
351
            prompt: prompt_to_string(&request.inner.prompt),
352
353
354
355
356
357
358
            system_prompt: None,
        });

        Ok(common::CompletionRequest {
            prompt,
            stop_conditions,
            sampling_options,
Greg Clark's avatar
Greg Clark committed
359
            output_options,
360
361
362
363
364
365
            mdc_sum: None,
            annotations: None,
        })
    }
}

366
impl TryFrom<common::StreamingCompletionResponse> for dynamo_protocols::types::Choice {
367
368
369
    type Error = anyhow::Error;

    fn try_from(response: common::StreamingCompletionResponse) -> Result<Self, Self::Error> {
370
371
372
373
374
        let text = response
            .delta
            .text
            .ok_or(anyhow::anyhow!("No text in response"))?;

375
        // SAFETY: we're downcasting from u64 to u32 here but u32::MAX is 4_294_967_295
376
        // so we're fairly safe knowing we won't generate that many Choices
377
378
379
380
381
382
        let index: u32 = response
            .delta
            .index
            .unwrap_or(0)
            .try_into()
            .expect("index exceeds u32::MAX");
383
384
385
386

        // TODO handle aggregating logprobs
        let logprobs = None;

387
        let finish_reason: Option<dynamo_protocols::types::CompletionFinishReason> =
388
389
            response.delta.finish_reason.map(Into::into);

390
        let choice = dynamo_protocols::types::Choice {
391
392
393
394
            text,
            index,
            logprobs,
            finish_reason,
395
396
397
398
399
        };

        Ok(choice)
    }
}
400

Greg Clark's avatar
Greg Clark committed
401
402
403
404
405
406
407
408
409
410
411
412
impl OpenAIOutputOptionsProvider for NvCreateCompletionRequest {
    fn get_logprobs(&self) -> Option<u32> {
        self.inner.logprobs.map(|logprobs| logprobs as u32)
    }

    fn get_prompt_logprobs(&self) -> Option<u32> {
        self.inner
            .echo
            .and_then(|echo| if echo { Some(1) } else { None })
    }

    fn get_skip_special_tokens(&self) -> Option<bool> {
413
        CommonExtProvider::get_skip_special_tokens(self)
Greg Clark's avatar
Greg Clark committed
414
415
416
417
418
419
420
    }

    fn get_formatted_prompt(&self) -> Option<bool> {
        None
    }
}

421
422
423
424
/// Implements `ValidateRequest` for `NvCreateCompletionRequest`,
/// allowing us to validate the data.
impl ValidateRequest for NvCreateCompletionRequest {
    fn validate(&self) -> Result<(), anyhow::Error> {
425
        validate::validate_no_unsupported_fields(&self.unsupported_fields)?;
426
        validate::validate_model(&self.inner.model)?;
427
428
429
430
431
432
433

        // Validate prompt and prompt_embeds together (checks presence, format, and content)
        validate::validate_prompt_or_embeds(
            Some(&self.inner.prompt),
            self.inner.prompt_embeds.as_deref(),
        )?;

434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
        validate::validate_suffix(self.inner.suffix.as_deref())?;
        validate::validate_max_tokens(self.inner.max_tokens)?;
        validate::validate_temperature(self.inner.temperature)?;
        validate::validate_top_p(self.inner.top_p)?;
        validate::validate_n(self.inner.n)?;
        // none for stream
        // none for stream_options
        validate::validate_logprobs(self.inner.logprobs)?;
        // none for echo
        validate::validate_stop(&self.inner.stop)?;
        validate::validate_presence_penalty(self.inner.presence_penalty)?;
        validate::validate_frequency_penalty(self.inner.frequency_penalty)?;
        validate::validate_best_of(self.inner.best_of, self.inner.n)?;
        validate::validate_logit_bias(&self.inner.logit_bias)?;
        validate::validate_user(self.inner.user.as_deref())?;
        // none for seed
450
        // none for metadata
451

452
453
        // Common Ext
        validate::validate_repetition_penalty(self.get_repetition_penalty())?;
454
455
        validate::validate_min_p(self.get_min_p())?;
        validate::validate_top_k(self.get_top_k())?;
456
457
        // Cross-field validation
        validate::validate_n_with_temperature(self.inner.n, self.inner.temperature)?;
458
459
460
461
462
        // total choices validation for completions batch requests
        validate::validate_total_choices(
            get_prompt_batch_size(&self.inner.prompt),
            self.inner.n.unwrap_or(1),
        )?;
463
464
465
        Ok(())
    }
}
466
467
468
469

#[cfg(test)]
mod tests {
    use super::*;
470
    use crate::engines::ValidateRequest;
471
    use crate::protocols::common::OutputOptionsProvider;
472
    use base64::Engine;
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
    use serde_json::json;

    #[test]
    fn test_skip_special_tokens_none() {
        let json_str = json!({
            "model": "test-model",
            "prompt": "Hello, world!"
        });

        let request: NvCreateCompletionRequest =
            serde_json::from_value(json_str).expect("Failed to deserialize request");

        assert_eq!(request.common.skip_special_tokens, None);

        let output_options = request
            .extract_output_options()
            .expect("Failed to extract output options");

        assert_eq!(output_options.skip_special_tokens, None);
    }

    #[test]
    fn test_skip_special_tokens_propagates() {
        for skip_value in [true, false] {
            let json_str = json!({
                "model": "test-model",
                "prompt": "Hello, world!",
                "skip_special_tokens": skip_value
            });

            let request: NvCreateCompletionRequest =
                serde_json::from_value(json_str).expect("Failed to deserialize request");

            let output_options = request
                .extract_output_options()
                .expect("Failed to extract output options");

            assert_eq!(output_options.skip_special_tokens, Some(skip_value));
        }
    }
513

514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
    #[test]
    fn test_prompt_embeds_only() {
        // Create valid embeddings: > 100 bytes (PyTorch format)
        let valid_data = vec![0u8; 256];
        let encoded = base64::engine::general_purpose::STANDARD.encode(&valid_data);

        let json_str = json!({
            "model": "test-model",
            "prompt": "test",
            "prompt_embeds": encoded
        });

        let request: NvCreateCompletionRequest =
            serde_json::from_value(json_str).expect("Failed to deserialize request");

        assert!(ValidateRequest::validate(&request).is_ok());
        assert!(request.inner.prompt_embeds.is_some());
    }

    #[test]
    fn test_both_prompt_and_embeds() {
        // Both fields are allowed, prompt_embeds takes precedence at worker level
        // Create valid embeddings: > 100 bytes (PyTorch format)
        let valid_data = vec![0u8; 256];
        let encoded = base64::engine::general_purpose::STANDARD.encode(&valid_data);

        let json_str = json!({
            "model": "test-model",
            "prompt": "Hello",
            "prompt_embeds": encoded
        });

        let request: NvCreateCompletionRequest =
            serde_json::from_value(json_str).expect("Failed to deserialize request");

        assert!(ValidateRequest::validate(&request).is_ok());
    }

    #[test]
    fn test_invalid_base64() {
        // Create invalid base64 that's long enough (>100 bytes) to pass size check
        // Use characters that look like base64 but aren't valid
        let invalid_base64 = "not-valid-base64!!!".repeat(10); // 190 bytes, looks like base64 but invalid

        let json_str = json!({
            "model": "test-model",
            "prompt": "test",
            "prompt_embeds": invalid_base64
        });

        let request: NvCreateCompletionRequest =
            serde_json::from_value(json_str).expect("Failed to deserialize request");

        let result = ValidateRequest::validate(&request);
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("base64"));
    }

    #[test]
    fn test_embeds_too_large() {
        // Create embeddings with DECODED size larger than 10MB
        // Base64 encoding adds ~33% overhead, so we need 11MB decoded = ~14.7MB encoded
        let large_data = vec![0u8; 11 * 1024 * 1024]; // 11MB decoded
        let large_embeds = base64::engine::general_purpose::STANDARD.encode(&large_data);

        let json_str = json!({
            "model": "test-model",
            "prompt": "test",
            "prompt_embeds": large_embeds
        });

        let request: NvCreateCompletionRequest =
            serde_json::from_value(json_str).expect("Failed to deserialize request");

        let result = ValidateRequest::validate(&request);
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("10MB"));
    }

    #[test]
    fn test_embeds_too_small() {
        // Create embeddings with DECODED size smaller than 100 bytes
        let small_data = vec![0u8; 20]; // Only 20 bytes when decoded
        let encoded = base64::engine::general_purpose::STANDARD.encode(&small_data);

        let json_str = json!({
            "model": "test-model",
            "prompt": "test",
            "prompt_embeds": encoded
        });

        let request: NvCreateCompletionRequest =
            serde_json::from_value(json_str).expect("Failed to deserialize request");

        let result = ValidateRequest::validate(&request);
        assert!(result.is_err());
        let err_msg = result.unwrap_err().to_string();
        assert!(
            err_msg.contains("100 bytes")
                || err_msg.contains("at least")
                || err_msg.contains("decoded")
        );
    }

    #[test]
    fn test_embeddings_with_empty_prompt() {
        // Test that empty prompt is ALLOWED when embeddings provided
        let valid_data = vec![0u8; 256]; // Valid size and aligned
        let encoded = base64::engine::general_purpose::STANDARD.encode(&valid_data);

        let json_str = json!({
            "model": "test-model",
            "prompt": "", // Empty prompt is OK with embeddings
            "prompt_embeds": encoded
        });

        let request: NvCreateCompletionRequest =
            serde_json::from_value(json_str).expect("Failed to deserialize request");

        // Should succeed - embeddings take precedence, prompt can be empty
        assert!(ValidateRequest::validate(&request).is_ok());
    }

    #[test]
    fn test_empty_prompt_without_embeddings_fails() {
        // Empty prompt WITHOUT embeddings should fail
        let json_str = json!({
            "model": "test-model",
            "prompt": "",  // Empty prompt
            // No prompt_embeds
        });

        let request: NvCreateCompletionRequest =
            serde_json::from_value(json_str).expect("Failed to deserialize request");

        let result = ValidateRequest::validate(&request);
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("cannot be empty"));
    }

654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
    #[test]
    fn test_stop() {
        let null_stop = json!({
            "model": "test-model",
            "prompt": "Hello, world!"
        });
        let request: NvCreateCompletionRequest =
            serde_json::from_value(null_stop).expect("Failed to deserialize request");
        assert_eq!(request.get_stop(), None);

        let one_stop = json!({
            "model": "test-model",
            "prompt": "Hello, world!",
            "stop": "foo"
        });
        let request: NvCreateCompletionRequest =
            serde_json::from_value(one_stop).expect("Failed to deserialize request");
        assert_eq!(request.get_stop(), Some(vec!["foo".to_string()]));

        let many_stops = json!({
            "model": "test-model",
            "prompt": "Hello, world!",
            "stop": ["foo", "bar"]
        });
        let request: NvCreateCompletionRequest =
            serde_json::from_value(many_stops).expect("Failed to deserialize request");
        assert_eq!(
            request.get_stop(),
            Some(vec!["foo".to_string(), "bar".to_string()])
        );
    }
685
}