Skip to content

GitLab

Unverified Commit b6a3b0c6 authored by ishandhanani's avatar ishandhanani Committed by GitHub
Browse files

refactor(2/3): rename dynamo-async-openai to dynamo-protocols (#7565)

parent c84c0934
Show whitespace changes
Inline Side-by-side
Showing with 144 additions and 156 deletions
Cargo.lock
View file @ b6a3b0c6
......@@ -1851,35 +1851,6 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "dynamo-async-openai"
version = "1.0.0"
dependencies = [
"async-openai-macros",
"backoff",
"base64 0.22.1",
"bytes",
"derive_builder",
"eventsource-stream",
"futures",
"rand 0.9.2",
"reqwest 0.12.28",
"reqwest-eventsource",
"secrecy",
"serde",
"serde_json",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tokio-test",
"tokio-tungstenite",
"tokio-util",
"tracing",
"url",
"utoipa",
"uuid",
]
[[package]]
name = "dynamo-bench"
version = "1.0.0"
......@@ -1983,12 +1954,12 @@ dependencies = [
"derive-getters",
"derive_builder",
"dialoguer",
"dynamo-async-openai",
"dynamo-bench",
"dynamo-kv-router",
"dynamo-memory",
"dynamo-mocker",
"dynamo-parsers",
"dynamo-protocols",
"dynamo-runtime",
"dynamo-tokens",
"either",
......@@ -2106,7 +2077,7 @@ name = "dynamo-parsers"
version = "1.0.0"
dependencies = [
"anyhow",
"dynamo-async-openai",
"dynamo-protocols",
"num-traits",
"openai-harmony",
"regex",
......@@ -2119,6 +2090,35 @@ dependencies = [
"uuid",
]
[[package]]
name = "dynamo-protocols"
version = "1.0.0"
dependencies = [
"async-openai-macros",
"backoff",
"base64 0.22.1",
"bytes",
"derive_builder",
"eventsource-stream",
"futures",
"rand 0.9.2",
"reqwest 0.12.28",
"reqwest-eventsource",
"secrecy",
"serde",
"serde_json",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tokio-test",
"tokio-tungstenite",
"tokio-util",
"tracing",
"url",
"utoipa",
"uuid",
]
[[package]]
name = "dynamo-runtime"
version = "1.0.0"
......
Cargo.toml
View file @ b6a3b0c6
......@@ -14,7 +14,7 @@ members = [
"lib/kvbm-kernels",
"lib/kvbm-logical",
"lib/kvbm-physical",
"lib/async-openai",
"lib/protocols",
"lib/parsers",
"lib/bench",
"lib/bindings/c",
......@@ -44,7 +44,7 @@ dynamo-tokens = { path = "lib/tokens", version = "1.0.0" }
dynamo-memory = { path = "lib/memory", version = "1.0.0" }
dynamo-mocker = { path = "lib/mocker", version = "1.0.0" }
dynamo-kv-router = { path = "lib/kv-router", version = "1.0.0", features = ["metrics", "runtime-protocols"] }
dynamo-async-openai = { path = "lib/async-openai", version = "1.0.0", features = ["byot"] }
dynamo-protocols = { path = "lib/protocols", version = "1.0.0", features = ["byot"] }
dynamo-parsers = { path = "lib/parsers", version = "1.0.0" }
fastokens = { version = "0.1.0" }
......
docs/reference/release-artifacts.md
View file @ b6a3b0c6
......@@ -61,7 +61,7 @@ We recommend using the TensorRT-LLM NGC container instead of the `ai-dynamo[trtl
|-------|-------------|-------------|-----------|
| `dynamo-runtime@1.0.1` | Core distributed runtime library | `v1.82` | [link](https://crates.io/crates/dynamo-runtime/1.0.1) |
| `dynamo-llm@1.0.1` | LLM inference engine | `v1.82` | [link](https://crates.io/crates/dynamo-llm/1.0.1) |
| `dynamo-async-openai@1.0.1` | Async OpenAI-compatible API client | `v1.82` | [link](https://crates.io/crates/dynamo-async-openai/1.0.1) |
| `dynamo-protocols@1.0.1` | Async OpenAI-compatible API client | `v1.82` | [link](https://crates.io/crates/dynamo-protocols/1.0.1) |
| `dynamo-parsers@1.0.1` | Protocol parsers (SSE, JSON streaming) | `v1.82` | [link](https://crates.io/crates/dynamo-parsers/1.0.1) |
| `dynamo-memory@1.0.1` | Memory management utilities | `v1.82` | [link](https://crates.io/crates/dynamo-memory/1.0.1) |
| `dynamo-config@1.0.1` | Configuration management | `v1.82` | [link](https://crates.io/crates/dynamo-config/1.0.1) |
......@@ -133,7 +133,7 @@ helm install snapshot oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/snapshot
```bash
cargo add dynamo-runtime@1.0.1
cargo add dynamo-llm@1.0.1
cargo add dynamo-async-openai@1.0.1
cargo add dynamo-protocols@1.0.1
cargo add dynamo-parsers@1.0.1
cargo add dynamo-memory@1.0.1
cargo add dynamo-config@1.0.1
......@@ -419,7 +419,7 @@ For a complete list of known issues, refer to the release notes for each version
### Rust Crates
> **crates.io:** [dynamo-runtime](https://crates.io/crates/dynamo-runtime) | [dynamo-llm](https://crates.io/crates/dynamo-llm) | [dynamo-async-openai](https://crates.io/crates/dynamo-async-openai) | [dynamo-parsers](https://crates.io/crates/dynamo-parsers) | [dynamo-memory](https://crates.io/crates/dynamo-memory) | [dynamo-config](https://crates.io/crates/dynamo-config) | [dynamo-tokens](https://crates.io/crates/dynamo-tokens)
> **crates.io:** [dynamo-runtime](https://crates.io/crates/dynamo-runtime) | [dynamo-llm](https://crates.io/crates/dynamo-llm) | [dynamo-protocols](https://crates.io/crates/dynamo-protocols) | [dynamo-parsers](https://crates.io/crates/dynamo-parsers) | [dynamo-memory](https://crates.io/crates/dynamo-memory) | [dynamo-config](https://crates.io/crates/dynamo-config) | [dynamo-tokens](https://crates.io/crates/dynamo-tokens)
>
> To access a specific version: `https://crates.io/crates/{crate}/{version}`
......@@ -453,20 +453,20 @@ For a complete list of known issues, refer to the release notes for each version
| `dynamo-llm@0.6.1` | `v1.82` | |
| `dynamo-llm@0.6.0` | `v1.82` | |
#### dynamo-async-openai (crate)
#### dynamo-protocols (crate)
| Crate | MSRV (Rust) | Notes |
|-------|-------------|-------|
| `dynamo-async-openai@1.0.1` | `v1.82` | |
| `dynamo-async-openai@1.0.0` | `v1.82` | |
| `dynamo-async-openai@0.9.1` | `v1.82` | |
| `dynamo-async-openai@0.9.0` | `v1.82` | |
| `dynamo-async-openai@0.8.1` | `v1.82` | |
| `dynamo-async-openai@0.8.0` | `v1.82` | |
| `dynamo-async-openai@0.7.1` | `v1.82` | |
| `dynamo-async-openai@0.7.0` | `v1.82` | |
| `dynamo-async-openai@0.6.1` | `v1.82` | |
| `dynamo-async-openai@0.6.0` | `v1.82` | |
| `dynamo-protocols@1.0.1` | `v1.82` | |
| `dynamo-protocols@1.0.0` | `v1.82` | |
| `dynamo-protocols@0.9.1` | `v1.82` | |
| `dynamo-protocols@0.9.0` | `v1.82` | |
| `dynamo-protocols@0.8.1` | `v1.82` | |
| `dynamo-protocols@0.8.0` | `v1.82` | |
| `dynamo-protocols@0.7.1` | `v1.82` | |
| `dynamo-protocols@0.7.0` | `v1.82` | |
| `dynamo-protocols@0.6.1` | `v1.82` | |
| `dynamo-protocols@0.6.0` | `v1.82` | |
#### dynamo-parsers (crate)
......
docs/reference/support-matrix.md
View file @ b6a3b0c6
......@@ -193,7 +193,7 @@ For version-specific artifact details, installation commands, and release histor
- **Rust Crates**:
- [dynamo-runtime](https://crates.io/crates/dynamo-runtime/)
- [dynamo-llm](https://crates.io/crates/dynamo-llm/)
- [dynamo-async-openai](https://crates.io/crates/dynamo-async-openai/)
- [dynamo-protocols](https://crates.io/crates/dynamo-protocols/)
- [dynamo-parsers](https://crates.io/crates/dynamo-parsers/)
- [dynamo-config](https://crates.io/crates/dynamo-config/) *(New in v0.8.0)*
- [dynamo-memory](https://crates.io/crates/dynamo-memory/) *(New in v0.8.0)*
......
lib/bindings/kvbm/Cargo.lock
View file @ b6a3b0c6
......@@ -1535,7 +1535,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "dynamo-async-openai"
name = "dynamo-protocols"
version = "1.0.0"
dependencies = [
"async-openai-macros",
......@@ -1621,7 +1621,7 @@ dependencies = [
"derive-getters",
"derive_builder",
"dialoguer",
"dynamo-async-openai",
"dynamo-protocols",
"dynamo-kv-router",
"dynamo-memory",
"dynamo-mocker",
......@@ -1729,7 +1729,7 @@ name = "dynamo-parsers"
version = "1.0.0"
dependencies = [
"anyhow",
"dynamo-async-openai",
"dynamo-protocols",
"num-traits",
"openai-harmony",
"regex",
......
lib/bindings/python/Cargo.lock
View file @ b6a3b0c6
......@@ -1543,7 +1543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "dynamo-async-openai"
name = "dynamo-protocols"
version = "1.0.0"
dependencies = [
"async-openai-macros",
......@@ -1634,7 +1634,7 @@ dependencies = [
"derive-getters",
"derive_builder",
"dialoguer",
"dynamo-async-openai",
"dynamo-protocols",
"dynamo-kv-router",
"dynamo-memory",
"dynamo-mocker",
......@@ -1745,7 +1745,7 @@ name = "dynamo-parsers"
version = "1.0.0"
dependencies = [
"anyhow",
"dynamo-async-openai",
"dynamo-protocols",
"num-traits",
"openai-harmony",
"regex",
......
lib/llm/Cargo.toml
View file @ b6a3b0c6
......@@ -56,7 +56,7 @@ dynamo-tokens = { workspace = true }
# workspace
aho-corasick = "1.1"
anyhow = { workspace = true }
dynamo-async-openai = { workspace = true }
dynamo-protocols = { workspace = true }
dynamo-parsers = { workspace = true }
async-stream = { workspace = true }
async-trait = { workspace = true }
......
lib/llm/src/audit/stream.rs
View file @ b6a3b0c6
......@@ -12,7 +12,7 @@ use crate::protocols::openai::chat_completions::{
};
use dynamo_runtime::protocols::annotated::Annotated;
use dynamo_async_openai::types::{ChatChoiceStream, ChatCompletionStreamResponseDelta};
use dynamo_protocols::types::{ChatChoiceStream, ChatCompletionStreamResponseDelta};
use futures::StreamExt;
type AuditStream =
......@@ -90,7 +90,7 @@ where
tracing::warn!("audit: aggregation future canceled/failed");
// Return minimal response if aggregation failed
NvCreateChatCompletionResponse {
inner: dynamo_async_openai::types::CreateChatCompletionResponse {
inner: dynamo_protocols::types::CreateChatCompletionResponse {
id: String::new(),
created: 0,
usage: None,
......@@ -127,7 +127,7 @@ where
Err(e) => {
tracing::warn!("fold aggregation failed: {e}");
let fallback = NvCreateChatCompletionResponse {
inner: dynamo_async_openai::types::CreateChatCompletionResponse {
inner: dynamo_protocols::types::CreateChatCompletionResponse {
id: String::new(),
created: 0,
usage: None,
......@@ -149,7 +149,7 @@ where
rx.await.unwrap_or_else(|_| {
tracing::warn!("fold aggregation future canceled");
NvCreateChatCompletionResponse {
inner: dynamo_async_openai::types::CreateChatCompletionResponse {
inner: dynamo_protocols::types::CreateChatCompletionResponse {
id: String::new(),
created: 0,
usage: None,
......@@ -182,7 +182,7 @@ pub fn final_response_to_one_chunk_stream(
// Convert FunctionCall to FunctionCallStream if present
#[allow(deprecated)]
let function_call = ch.message.function_call.as_ref().map(|fc| {
dynamo_async_openai::types::FunctionCallStream {
dynamo_protocols::types::FunctionCallStream {
name: Some(fc.name.clone()),
arguments: Some(fc.arguments.clone()),
}
......@@ -194,11 +194,11 @@ pub fn final_response_to_one_chunk_stream(
.iter()
.enumerate()
.map(
|(i, call)| dynamo_async_openai::types::ChatCompletionMessageToolCallChunk {
|(i, call)| dynamo_protocols::types::ChatCompletionMessageToolCallChunk {
index: i as u32,
id: Some(call.id.clone()),
r#type: Some(call.r#type.clone()),
function: Some(dynamo_async_openai::types::FunctionCallStream {
function: Some(dynamo_protocols::types::FunctionCallStream {
name: Some(call.function.name.clone()),
arguments: Some(call.function.arguments.clone()),
}),
......@@ -228,7 +228,7 @@ pub fn final_response_to_one_chunk_stream(
}
let chunk = NvCreateChatCompletionStreamResponse {
inner: dynamo_async_openai::types::CreateChatCompletionStreamResponse {
inner: dynamo_protocols::types::CreateChatCompletionStreamResponse {
id: resp.inner.id.clone(),
object: "chat.completion.chunk".to_string(),
created: resp.inner.created,
......@@ -254,7 +254,7 @@ pub fn final_response_to_one_chunk_stream(
#[cfg(test)]
mod tests {
use super::*;
use dynamo_async_openai::types::{
use dynamo_protocols::types::{
ChatChoiceStream, ChatCompletionMessageContent, ChatCompletionStreamResponseDelta,
FinishReason, Role,
};
......@@ -283,7 +283,7 @@ mod tests {
};
let response = NvCreateChatCompletionStreamResponse {
inner: dynamo_async_openai::types::CreateChatCompletionStreamResponse {
inner: dynamo_protocols::types::CreateChatCompletionStreamResponse {
id: "test-id".to_string(),
choices: vec![choice],
created: 1234567890,
......@@ -324,7 +324,7 @@ mod tests {
};
let response = NvCreateChatCompletionStreamResponse {
inner: dynamo_async_openai::types::CreateChatCompletionStreamResponse {
inner: dynamo_protocols::types::CreateChatCompletionStreamResponse {
id: "test-id".to_string(),
choices: vec![choice],
created: 1234567890,
......@@ -435,7 +435,7 @@ mod tests {
// Test that metadata (id, event, comment) is preserved through passthrough
let chunk_with_metadata = Annotated {
data: Some(NvCreateChatCompletionStreamResponse {
inner: dynamo_async_openai::types::CreateChatCompletionStreamResponse {
inner: dynamo_protocols::types::CreateChatCompletionStreamResponse {
id: "test-id".to_string(),
choices: vec![{
#[allow(deprecated)]
......
lib/llm/src/backend.rs
View file @ b6a3b0c6
......@@ -43,7 +43,7 @@ use crate::protocols::{
},
};
use crate::tokenizers::{DecodeStream, Tokenizer};
use dynamo_async_openai::types::StopReason;
use dynamo_protocols::types::StopReason;
/// Represents the output stream from the execution engine
pub type ExecutionOutputStream = Annotated<LLMEngineOutput>;
......
lib/llm/src/engines.rs
View file @ b6a3b0c6
......@@ -143,9 +143,9 @@ impl
};
let prompt = match req {
dynamo_async_openai::types::ChatCompletionRequestMessage::User(user_msg) => {
dynamo_protocols::types::ChatCompletionRequestMessage::User(user_msg) => {
match user_msg.content {
dynamo_async_openai::types::ChatCompletionRequestUserMessageContent::Text(
dynamo_protocols::types::ChatCompletionRequestUserMessageContent::Text(
prompt,
) => prompt,
_ => anyhow::bail!("Invalid request content field, expected Content::Text"),
......@@ -164,7 +164,7 @@ impl
id += 1;
}
let response = deltas.create_choice(0, None, Some(dynamo_async_openai::types::FinishReason::Stop), None, None);
let response = deltas.create_choice(0, None, Some(dynamo_protocols::types::FinishReason::Stop), None, None);
yield Annotated { id: Some(id.to_string()), data: Some(response), event: None, comment: None, error: None };
};
......@@ -196,7 +196,7 @@ impl
yield Annotated{ id: Some(id.to_string()), data: Some(response), event: None, comment: None, error: None };
id += 1;
}
let response = deltas.create_choice(0, None, Some(dynamo_async_openai::types::CompletionFinishReason::Stop), None);
let response = deltas.create_choice(0, None, Some(dynamo_protocols::types::CompletionFinishReason::Stop), None);
yield Annotated { id: Some(id.to_string()), data: Some(response), event: None, comment: None, error: None };
};
......
lib/llm/src/entrypoint/input/batch.rs
View file @ b6a3b0c6
......@@ -7,7 +7,7 @@ use crate::types::openai::chat_completions::{
NvCreateChatCompletionRequest, OpenAIChatCompletionsStreamingEngine,
};
use anyhow::Context as _;
use dynamo_async_openai::types::{ChatCompletionMessageContent, FinishReason};
use dynamo_protocols::types::{ChatCompletionMessageContent, FinishReason};
use dynamo_runtime::{DistributedRuntime, pipeline::Context, runtime::CancellationToken};
use futures::StreamExt;
use serde::{Deserialize, Serialize};
......@@ -201,15 +201,15 @@ async fn evaluate(
entry: &mut Entry,
template: Option<Arc<RequestTemplate>>,
) -> anyhow::Result<String> {
let user_message = dynamo_async_openai::types::ChatCompletionRequestMessage::User(
dynamo_async_openai::types::ChatCompletionRequestUserMessage {
content: dynamo_async_openai::types::ChatCompletionRequestUserMessageContent::Text(
let user_message = dynamo_protocols::types::ChatCompletionRequestMessage::User(
dynamo_protocols::types::ChatCompletionRequestUserMessage {
content: dynamo_protocols::types::ChatCompletionRequestUserMessageContent::Text(
entry.text.clone(),
),
name: None,
},
);
let inner = dynamo_async_openai::types::CreateChatCompletionRequestArgs::default()
let inner = dynamo_protocols::types::CreateChatCompletionRequestArgs::default()
.messages(vec![user_message])
.model(
template
......
lib/llm/src/entrypoint/input/text.rs
View file @ b6a3b0c6
......@@ -7,7 +7,7 @@ use crate::request_template::RequestTemplate;
use crate::types::openai::chat_completions::{
NvCreateChatCompletionRequest, OpenAIChatCompletionsStreamingEngine,
};
use dynamo_async_openai::types::ChatCompletionMessageContent;
use dynamo_protocols::types::ChatCompletionMessageContent;
use dynamo_runtime::DistributedRuntime;
use dynamo_runtime::pipeline::Context;
use futures::StreamExt;
......@@ -82,9 +82,9 @@ async fn main_loop(
};
// Construct messages
let user_message = dynamo_async_openai::types::ChatCompletionRequestMessage::User(
dynamo_async_openai::types::ChatCompletionRequestUserMessage {
content: dynamo_async_openai::types::ChatCompletionRequestUserMessageContent::Text(
let user_message = dynamo_protocols::types::ChatCompletionRequestMessage::User(
dynamo_protocols::types::ChatCompletionRequestUserMessage {
content: dynamo_protocols::types::ChatCompletionRequestUserMessageContent::Text(
prompt,
),
name: None,
......@@ -92,7 +92,7 @@ async fn main_loop(
);
messages.push(user_message);
// Request
let inner = dynamo_async_openai::types::CreateChatCompletionRequestArgs::default()
let inner = dynamo_protocols::types::CreateChatCompletionRequestArgs::default()
.messages(messages.clone())
.model(
template
......@@ -181,15 +181,15 @@ async fn main_loop(
println!();
let assistant_content =
dynamo_async_openai::types::ChatCompletionRequestAssistantMessageContent::Text(
dynamo_protocols::types::ChatCompletionRequestAssistantMessageContent::Text(
assistant_message,
);
let assistant_message = dynamo_async_openai::types::ChatCompletionRequestMessage::Assistant(
dynamo_async_openai::types::ChatCompletionRequestAssistantMessage {
let assistant_message = dynamo_protocols::types::ChatCompletionRequestMessage::Assistant(
dynamo_protocols::types::ChatCompletionRequestAssistantMessage {
content: Some(assistant_content),
reasoning_content: (!assistant_reasoning.is_empty()).then_some(
dynamo_async_openai::types::ReasoningContent::Text(assistant_reasoning),
dynamo_protocols::types::ReasoningContent::Text(assistant_reasoning),
),
..Default::default()
},
......
lib/llm/src/grpc/service/openai.rs
View file @ b6a3b0c6
......@@ -23,7 +23,7 @@ use crate::http::service::{
disconnect::{ConnectionHandle, create_connection_monitor},
metrics::{CancellationLabels, Endpoint, InflightGuard, process_response_and_observe_metrics},
};
use dynamo_async_openai::types::{CompletionFinishReason, CreateCompletionRequest, Prompt};
use dynamo_protocols::types::{CompletionFinishReason, CreateCompletionRequest, Prompt};
use tonic::Status;
......
lib/llm/src/http/client.rs
View file @ b6a3b0c6
......@@ -14,7 +14,7 @@ use std::time::Instant;
use async_trait::async_trait;
use derive_getters::Dissolve;
use dynamo_async_openai::{Client, config::OpenAIConfig, error::OpenAIError};
use dynamo_protocols::{Client, config::OpenAIConfig, error::OpenAIError};
use futures::Stream;
use serde_json::Value;
use tokio_util::sync::CancellationToken;
......@@ -285,7 +285,7 @@ pub type ByotResponseStream = DataStream<Result<Value, OpenAIError>>;
/// Type alias for pure OpenAI chat response stream
pub type OpenAIChatResponseStream =
DataStream<Result<dynamo_async_openai::types::CreateChatCompletionStreamResponse, OpenAIError>>;
DataStream<Result<dynamo_protocols::types::CreateChatCompletionStreamResponse, OpenAIError>>;
/// A wrapped HTTP response stream that combines a stream with its context
/// This provides a unified interface for HTTP client responses
......@@ -381,7 +381,7 @@ pub type ByotHttpResponseStream = HttpResponseStream<Result<Value, OpenAIError>>
/// Type alias for HttpResponseStream with pure OpenAI responses
pub type OpenAIHttpResponseStream = HttpResponseStream<
Result<dynamo_async_openai::types::CreateChatCompletionStreamResponse, OpenAIError>,
Result<dynamo_protocols::types::CreateChatCompletionStreamResponse, OpenAIError>,
>;
/// Pure OpenAI client using standard async-openai types
......@@ -401,7 +401,7 @@ impl PureOpenAIClient {
/// Uses a client-managed context
pub async fn chat_stream(
&self,
request: dynamo_async_openai::types::CreateChatCompletionRequest,
request: dynamo_protocols::types::CreateChatCompletionRequest,
) -> Result<OpenAIHttpResponseStream, HttpClientError> {
let ctx = self.base.create_context();
self.chat_stream_with_context(request, ctx).await
......@@ -410,7 +410,7 @@ impl PureOpenAIClient {
/// Create streaming chat completions with a custom context
pub async fn chat_stream_with_context(
&self,
request: dynamo_async_openai::types::CreateChatCompletionRequest,
request: dynamo_protocols::types::CreateChatCompletionRequest,
context: HttpRequestContext,
) -> Result<OpenAIHttpResponseStream, HttpClientError> {
let ctx_arc: Arc<dyn AsyncEngineContext> = Arc::new(context.clone());
......
lib/llm/src/http/service/openai.rs
View file @ b6a3b0c6
......@@ -1538,7 +1538,7 @@ async fn responses(
// Set stream_options.include_usage so the backend sends token counts in the final chunk.
chat_request.inner.stream = Some(true);
chat_request.inner.stream_options =
Some(dynamo_async_openai::types::ChatCompletionStreamOptions {
Some(dynamo_protocols::types::ChatCompletionStreamOptions {
include_usage: true,
continuous_usage_stats: false,
});
......@@ -1915,9 +1915,9 @@ async fn images(
.model
.as_ref()
.map(|m| match m {
dynamo_async_openai::types::ImageModel::DallE2 => "dall-e-2".to_string(),
dynamo_async_openai::types::ImageModel::DallE3 => "dall-e-3".to_string(),
dynamo_async_openai::types::ImageModel::Other(s) => s.clone(),
dynamo_protocols::types::ImageModel::DallE2 => "dall-e-2".to_string(),
dynamo_protocols::types::ImageModel::DallE3 => "dall-e-3".to_string(),
dynamo_protocols::types::ImageModel::Other(s) => s.clone(),
})
.unwrap_or_else(|| "diffusion".to_string());
......@@ -2327,8 +2327,8 @@ mod tests {
use crate::protocols::openai::common_ext::CommonExt;
use crate::protocols::openai::completions::NvCreateCompletionRequest;
use crate::protocols::openai::responses::NvCreateResponse;
use dynamo_async_openai::types::responses::{CreateResponse, Input, PromptConfig};
use dynamo_async_openai::types::{
use dynamo_protocols::types::responses::{CreateResponse, Input, PromptConfig};
use dynamo_protocols::types::{
ChatCompletionRequestMessage, ChatCompletionRequestUserMessage,
ChatCompletionRequestUserMessageContent, CreateChatCompletionRequest,
CreateCompletionRequest,
......@@ -3004,7 +3004,7 @@ mod tests {
#[tokio::test]
async fn test_check_for_backend_error_with_normal_event() {
use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
use dynamo_async_openai::types::CreateChatCompletionStreamResponse;
use dynamo_protocols::types::CreateChatCompletionStreamResponse;
use futures::stream::{self, StreamExt};
// Create a normal data event
......@@ -3178,7 +3178,7 @@ mod tests {
use std::collections::{HashMap, HashSet};
use dynamo_async_openai::types::{
use dynamo_protocols::types::{
ChatChoiceStream, ChatCompletionMessageToolCallChunk, ChatCompletionStreamResponseDelta,
ChatCompletionToolType, CreateChatCompletionStreamResponse, FinishReason,
FunctionCallStream,
......
lib/llm/src/perf/logprobs.rs
View file @ b6a3b0c6
......@@ -572,7 +572,7 @@ mod tests {
use crate::protocols::codec::create_message_stream;
use crate::protocols::convert_sse_stream;
use approx::assert_abs_diff_eq;
use dynamo_async_openai::types::{
use dynamo_protocols::types::{
ChatChoiceLogprobs, ChatChoiceStream, ChatCompletionStreamResponseDelta,
ChatCompletionTokenLogprob, FinishReason, Role, TopLogprobs,
};
......@@ -949,16 +949,14 @@ mod tests {
) -> NvCreateChatCompletionStreamResponse {
#[expect(deprecated)]
NvCreateChatCompletionStreamResponse {
inner: dynamo_async_openai::types::CreateChatCompletionStreamResponse {
inner: dynamo_protocols::types::CreateChatCompletionStreamResponse {
id: "test_id".to_string(),
choices: vec![ChatChoiceStream {
index: 0,
delta: ChatCompletionStreamResponseDelta {
content: Some(
dynamo_async_openai::types::ChatCompletionMessageContent::Text(
content: Some(dynamo_protocols::types::ChatCompletionMessageContent::Text(
"test".to_string(),
),
),
)),
function_call: None,
tool_calls: None,
role: Some(Role::Assistant),
......@@ -993,11 +991,9 @@ mod tests {
.map(|(i, token_logprobs)| ChatChoiceStream {
index: i as u32,
delta: ChatCompletionStreamResponseDelta {
content: Some(
dynamo_async_openai::types::ChatCompletionMessageContent::Text(
content: Some(dynamo_protocols::types::ChatCompletionMessageContent::Text(
"test".to_string(),
),
),
)),
function_call: None,
tool_calls: None,
role: Some(Role::Assistant),
......@@ -1014,7 +1010,7 @@ mod tests {
.collect();
NvCreateChatCompletionStreamResponse {
inner: dynamo_async_openai::types::CreateChatCompletionStreamResponse {
inner: dynamo_protocols::types::CreateChatCompletionStreamResponse {
id: "test_id".to_string(),
choices,
created: 1234567890,
......@@ -1345,16 +1341,14 @@ mod tests {
// Test with choice that has no logprobs
#[expect(deprecated)]
let response = NvCreateChatCompletionStreamResponse {
inner: dynamo_async_openai::types::CreateChatCompletionStreamResponse {
inner: dynamo_protocols::types::CreateChatCompletionStreamResponse {
id: "test_id".to_string(),
choices: vec![ChatChoiceStream {
index: 0,
delta: ChatCompletionStreamResponseDelta {
content: Some(
dynamo_async_openai::types::ChatCompletionMessageContent::Text(
content: Some(dynamo_protocols::types::ChatCompletionMessageContent::Text(
"test".to_string(),
),
),
)),
function_call: None,
tool_calls: None,
role: Some(Role::Assistant),
......@@ -1579,7 +1573,7 @@ mod tests {
// In practice, this would have real logprobs data
NvCreateChatCompletionStreamResponse {
inner: dynamo_async_openai::types::CreateChatCompletionStreamResponse {
inner: dynamo_protocols::types::CreateChatCompletionStreamResponse {
id: "test_id".to_string(),
choices: vec![],
created: 1234567890,
......
lib/llm/src/preprocessor.rs
View file @ b6a3b0c6
......@@ -18,7 +18,7 @@ pub mod tools;
use anyhow::Context;
use anyhow::{Result, bail};
use dynamo_async_openai::types::{
use dynamo_protocols::types::{
ChatCompletionRequestMessage, ChatCompletionRequestUserMessageContent,
ChatCompletionRequestUserMessageContentPart, ChatCompletionToolChoiceOption, EncodingFormat,
};
......@@ -649,11 +649,11 @@ impl OpenAIPreprocessor {
let mut builder = PreprocessedEmbeddingRequest::builder();
let all_token_ids = match &request.inner.input {
dynamo_async_openai::types::EmbeddingInput::String(s) => {
dynamo_protocols::types::EmbeddingInput::String(s) => {
let encoding = self.tokenizer.encode(s)?;
vec![encoding.token_ids().to_vec()]
}
dynamo_async_openai::types::EmbeddingInput::StringArray(arr) => {
dynamo_protocols::types::EmbeddingInput::StringArray(arr) => {
let input_strs: Vec<String> = arr.to_vec();
let encodings = tokio::task::spawn_blocking({
let tokenizer = self.tokenizer.clone();
......@@ -669,10 +669,10 @@ impl OpenAIPreprocessor {
.collect();
token_arrays
}
dynamo_async_openai::types::EmbeddingInput::IntegerArray(token_ids) => {
dynamo_protocols::types::EmbeddingInput::IntegerArray(token_ids) => {
vec![token_ids.clone()]
}
dynamo_async_openai::types::EmbeddingInput::ArrayOfIntegerArray(token_arrays) => {
dynamo_protocols::types::EmbeddingInput::ArrayOfIntegerArray(token_arrays) => {
token_arrays.clone()
}
};
......@@ -1032,11 +1032,11 @@ impl OpenAIPreprocessor {
stream.map(move |output| {
output.map_data(|engine_output| {
// Convert engine output to OpenAI response format
let embeddings: Vec<dynamo_async_openai::types::Embedding> = engine_output
let embeddings: Vec<dynamo_protocols::types::Embedding> = engine_output
.embeddings
.into_iter()
.enumerate()
.map(|(index, embedding)| dynamo_async_openai::types::Embedding {
.map(|(index, embedding)| dynamo_protocols::types::Embedding {
index: index as u32,
object: "embedding".to_string(),
embedding: embedding.into_iter().map(|f| f as f32).collect(),
......@@ -1044,11 +1044,11 @@ impl OpenAIPreprocessor {
.collect();
let response = NvCreateEmbeddingResponse {
inner: dynamo_async_openai::types::CreateEmbeddingResponse {
inner: dynamo_protocols::types::CreateEmbeddingResponse {
object: "list".to_string(),
model: original_request.inner.model.clone(),
data: embeddings,
usage: dynamo_async_openai::types::EmbeddingUsage {
usage: dynamo_protocols::types::EmbeddingUsage {
prompt_tokens: engine_output.prompt_tokens,
total_tokens: engine_output.total_tokens,
},
......@@ -1095,14 +1095,14 @@ impl OpenAIPreprocessor {
/// Apply tool calling jail to the stream if needed
pub fn apply_tool_calling_jail<S>(
tool_call_parser: Option<String>,
tool_choice: Option<dynamo_async_openai::types::ChatCompletionToolChoiceOption>,
tool_choice: Option<dynamo_protocols::types::ChatCompletionToolChoiceOption>,
tool_definitions: Option<Vec<dynamo_parsers::tool_calling::ToolDefinition>>,
stream: S,
) -> impl Stream<Item = Annotated<NvCreateChatCompletionStreamResponse>> + Send
where
S: Stream<Item = Annotated<NvCreateChatCompletionStreamResponse>> + Send + 'static,
{
use dynamo_async_openai::types::ChatCompletionToolChoiceOption;
use dynamo_protocols::types::ChatCompletionToolChoiceOption;
let mut builder = JailedStream::builder();
......@@ -1227,9 +1227,7 @@ impl OpenAIPreprocessor {
for choice in data.inner.choices.iter_mut() {
// Reasoning parsing only applies to text content
if let Some(
dynamo_async_openai::types::ChatCompletionMessageContent::Text(
text,
),
dynamo_protocols::types::ChatCompletionMessageContent::Text(text),
) = choice.delta.content.as_ref()
{
let parser_result =
......@@ -1237,7 +1235,7 @@ impl OpenAIPreprocessor {
// Update this specific choice with parsed content
choice.delta.content = parser_result.get_some_normal_text().map(
dynamo_async_openai::types::ChatCompletionMessageContent::Text,
dynamo_protocols::types::ChatCompletionMessageContent::Text,
);
choice.delta.reasoning_content = parser_result.get_some_reasoning();
}
......
lib/llm/src/preprocessor/media/loader.rs
View file @ b6a3b0c6
......@@ -6,8 +6,8 @@ use std::time::Duration;
use anyhow::Result;
use dynamo_async_openai::types::ChatCompletionRequestUserMessageContentPart;
use dynamo_memory::nixl::NixlAgent;
use dynamo_protocols::types::ChatCompletionRequestUserMessageContentPart;
use super::common::EncodedMediaData;
use super::decoders::{Decoder, MediaDecoder};
......@@ -157,7 +157,7 @@ mod tests {
use super::super::decoders::ImageDecoder;
use super::super::rdma::DataType;
use super::*;
use dynamo_async_openai::types::{ChatCompletionRequestMessageContentPartImage, ImageUrl};
use dynamo_protocols::types::{ChatCompletionRequestMessageContentPartImage, ImageUrl};
#[tokio::test]
async fn test_fetch_and_decode() {
......
lib/llm/src/preprocessor/prompt.rs
View file @ b6a3b0c6
......@@ -52,9 +52,7 @@ pub enum PromptInput {
pub trait OAIChatLikeRequest {
fn model(&self) -> String;
fn messages(&self) -> Value;
fn typed_messages(
&self,
) -> Option<&[dynamo_async_openai::types::ChatCompletionRequestMessage]> {
fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
None
}
fn tools(&self) -> Option<Value> {
......
lib/llm/src/preprocessor/prompt/template/oai.rs
View file @ b6a3b0c6
......@@ -289,9 +289,7 @@ impl OAIChatLikeRequest for NvCreateChatCompletionRequest {
Value::from_serialize(&messages_json)
}
fn typed_messages(
&self,
) -> Option<&[dynamo_async_openai::types::ChatCompletionRequestMessage]> {
fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
Some(self.inner.messages.as_slice())
}
......@@ -331,7 +329,7 @@ impl OAIChatLikeRequest for NvCreateChatCompletionRequest {
// .map(|last| {
// !matches!(
// last,
// dynamo_async_openai::types::ChatCompletionRequestMessage::Assistant(_)
// dynamo_protocols::types::ChatCompletionRequestMessage::Assistant(_)
// )
// })
// .unwrap_or(true)
......@@ -355,9 +353,9 @@ impl OAIChatLikeRequest for NvCreateCompletionRequest {
self.inner.model.clone()
}
fn messages(&self) -> minijinja::value::Value {
let message = dynamo_async_openai::types::ChatCompletionRequestMessage::User(
dynamo_async_openai::types::ChatCompletionRequestUserMessage {
content: dynamo_async_openai::types::ChatCompletionRequestUserMessageContent::Text(
let message = dynamo_protocols::types::ChatCompletionRequestMessage::User(
dynamo_protocols::types::ChatCompletionRequestUserMessage {
content: dynamo_protocols::types::ChatCompletionRequestUserMessageContent::Text(
crate::protocols::openai::completions::prompt_to_string(&self.inner.prompt),
),
name: None,
......@@ -373,16 +371,16 @@ impl OAIChatLikeRequest for NvCreateCompletionRequest {
fn prompt_input_type(&self) -> PromptInput {
match &self.inner.prompt {
dynamo_async_openai::types::Prompt::IntegerArray(_) => {
dynamo_protocols::types::Prompt::IntegerArray(_) => {
PromptInput::Tokens(TokenInput::Single(vec![]))
}
dynamo_async_openai::types::Prompt::ArrayOfIntegerArray(_) => {
dynamo_protocols::types::Prompt::ArrayOfIntegerArray(_) => {
PromptInput::Tokens(TokenInput::Batch(vec![]))
}
dynamo_async_openai::types::Prompt::String(_) => {
dynamo_protocols::types::Prompt::String(_) => {
PromptInput::Text(TextInput::Single(String::new()))
}
dynamo_async_openai::types::Prompt::StringArray(_) => {
dynamo_protocols::types::Prompt::StringArray(_) => {
PromptInput::Text(TextInput::Batch(vec![]))
}
}
......@@ -390,10 +388,10 @@ impl OAIChatLikeRequest for NvCreateCompletionRequest {
fn extract_tokens(&self) -> Option<TokenInput> {
match &self.inner.prompt {
dynamo_async_openai::types::Prompt::IntegerArray(tokens) => {
dynamo_protocols::types::Prompt::IntegerArray(tokens) => {
Some(TokenInput::Single(tokens.clone()))
}
dynamo_async_openai::types::Prompt::ArrayOfIntegerArray(arrays) => {
dynamo_protocols::types::Prompt::ArrayOfIntegerArray(arrays) => {
Some(TokenInput::Batch(arrays.clone()))
}
_ => None,
......@@ -402,10 +400,10 @@ impl OAIChatLikeRequest for NvCreateCompletionRequest {
fn extract_text(&self) -> Option<TextInput> {
match &self.inner.prompt {
dynamo_async_openai::types::Prompt::String(text) => {
dynamo_protocols::types::Prompt::String(text) => {
Some(TextInput::Single(text.to_string()))
}
dynamo_async_openai::types::Prompt::StringArray(texts) => {
dynamo_protocols::types::Prompt::StringArray(texts) => {
Some(TextInput::Batch(texts.to_vec()))
}
_ => None,
......@@ -492,7 +490,7 @@ impl OAIPromptFormatter for HfTokenizerConfigJsonFormatter {
#[cfg(test)]
mod tests {
use super::*;
use dynamo_async_openai::types::ChatCompletionRequestMessage as Msg;
use dynamo_protocols::types::ChatCompletionRequestMessage as Msg;
use minijinja::{Environment, context};
/// Tests that media URL content parts are converted to empty placeholders.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment