Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
dynamo
Commits
602352ce
Commit
602352ce
authored
Mar 08, 2025
by
Neelay Shah
Committed by
GitHub
Mar 08, 2025
Browse files
chore: rename dynamo (#44)
Co-authored-by:
Biswa Panda
<
biswa.panda@gmail.com
>
parent
ecf53ce2
Changes
431
Hide whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
39 additions
and
39 deletions
+39
-39
lib/llm/src/http/service/openai.rs
lib/llm/src/http/service/openai.rs
+3
-3
lib/llm/src/kv_router.rs
lib/llm/src/kv_router.rs
+3
-3
lib/llm/src/kv_router/metrics_aggregator.rs
lib/llm/src/kv_router/metrics_aggregator.rs
+2
-2
lib/llm/src/kv_router/publisher.rs
lib/llm/src/kv_router/publisher.rs
+1
-1
lib/llm/src/kv_router/scheduler.rs
lib/llm/src/kv_router/scheduler.rs
+2
-2
lib/llm/src/kv_router/worker.rs
lib/llm/src/kv_router/worker.rs
+2
-2
lib/llm/src/lib.rs
lib/llm/src/lib.rs
+2
-2
lib/llm/src/model_card/model.rs
lib/llm/src/model_card/model.rs
+1
-1
lib/llm/src/preprocessor.rs
lib/llm/src/preprocessor.rs
+3
-3
lib/llm/src/protocols.rs
lib/llm/src/protocols.rs
+5
-5
lib/llm/src/protocols/common/preprocessor.rs
lib/llm/src/protocols/common/preprocessor.rs
+1
-1
lib/llm/src/protocols/openai/chat_completions.rs
lib/llm/src/protocols/openai/chat_completions.rs
+1
-1
lib/llm/src/protocols/openai/completions.rs
lib/llm/src/protocols/openai/completions.rs
+1
-1
lib/llm/src/tokenizers/README.md
lib/llm/src/tokenizers/README.md
+3
-3
lib/llm/src/types.rs
lib/llm/src/types.rs
+1
-1
lib/llm/tests/aggregators.rs
lib/llm/tests/aggregators.rs
+1
-1
lib/llm/tests/backend.rs
lib/llm/tests/backend.rs
+2
-2
lib/llm/tests/http-service.rs
lib/llm/tests/http-service.rs
+3
-3
lib/llm/tests/model_card.rs
lib/llm/tests/model_card.rs
+1
-1
lib/llm/tests/openai_completions.rs
lib/llm/tests/openai_completions.rs
+1
-1
No files found.
lib/llm/src/http/service/openai.rs
View file @
602352ce
...
...
@@ -48,7 +48,7 @@ use crate::types::{
Annotated
,
};
use
dyn
e
mo_runtime
::
pipeline
::{
AsyncEngineContext
,
Context
};
use
dyn
a
mo_runtime
::
pipeline
::{
AsyncEngineContext
,
Context
};
#[derive(Serialize,
Deserialize)]
pub
(
crate
)
struct
ErrorResponse
{
...
...
@@ -91,7 +91,7 @@ impl ErrorResponse {
)
}
/// The OAI endpoints call an [`dyn
e
mo
_
runtime::engine::AsyncEngine`] which are specialized to return
/// The OAI endpoints call an [`dyn
a
mo
.
runtime::engine::AsyncEngine`] which are specialized to return
/// an [`anyhow::Error`]. This method will convert the [`anyhow::Error`] into an [`HttpError`].
/// If successful, it will return the [`HttpError`] as an [`ErrorResponse::internal_server_error`]
/// with the details of the error.
...
...
@@ -516,7 +516,7 @@ pub fn list_models_router(
path
:
Option
<
String
>
,
)
->
(
Vec
<
RouteDoc
>
,
Router
)
{
// TODO: Why do we have this endpoint?
let
custom_path
=
path
.unwrap_or
(
"/dyn
e
mo/alpha/list-models"
.to_string
());
let
custom_path
=
path
.unwrap_or
(
"/dyn
a
mo/alpha/list-models"
.to_string
());
let
doc_for_custom
=
RouteDoc
::
new
(
axum
::
http
::
Method
::
GET
,
&
custom_path
);
// Standard OpenAI compatible list models endpoint
...
...
lib/llm/src/kv_router.rs
View file @
602352ce
...
...
@@ -14,7 +14,7 @@
// limitations under the License.
use
anyhow
::
Result
;
use
dyn
e
mo_runtime
::{
component
::
Component
,
component
::
Namespace
,
DistributedRuntime
};
use
dyn
a
mo_runtime
::{
component
::
Component
,
component
::
Namespace
,
DistributedRuntime
};
use
futures
::
stream
::
StreamExt
;
use
std
::{
sync
::
Arc
,
time
::
Duration
};
use
tokio_util
::
sync
::
CancellationToken
;
...
...
@@ -67,7 +67,7 @@ impl KvRouter {
}
pub
async
fn
new
(
nats_client
:
dyn
e
mo_runtime
::
transports
::
nats
::
Client
,
nats_client
:
dyn
a
mo_runtime
::
transports
::
nats
::
Client
,
service_name
:
String
,
kv_subject
:
String
,
namespace
:
Namespace
,
...
...
@@ -141,7 +141,7 @@ impl KvRouter {
}
async
fn
collect_endpoints
(
nats_client
:
dyn
e
mo_runtime
::
transports
::
nats
::
Client
,
nats_client
:
dyn
a
mo_runtime
::
transports
::
nats
::
Client
,
service_name
:
String
,
ep_tx
:
tokio
::
sync
::
mpsc
::
Sender
<
ProcessedEndpoints
>
,
cancel
:
CancellationToken
,
...
...
lib/llm/src/kv_router/metrics_aggregator.rs
View file @
602352ce
...
...
@@ -19,7 +19,7 @@ pub use crate::kv_router::protocols::ForwardPassMetrics;
use
crate
::
kv_router
::
scheduler
::{
Endpoint
,
Service
};
use
crate
::
kv_router
::
ProcessedEndpoints
;
use
dyn
e
mo_runtime
::
component
::
Component
;
use
dyn
a
mo_runtime
::
component
::
Component
;
use
std
::
time
::
Duration
;
use
tokio_util
::
sync
::
CancellationToken
;
...
...
@@ -81,7 +81,7 @@ impl KvMetricsAggregator {
}
async
fn
collect_endpoints
(
nats_client
:
dyn
e
mo_runtime
::
transports
::
nats
::
Client
,
nats_client
:
dyn
a
mo_runtime
::
transports
::
nats
::
Client
,
service_name
:
String
,
ep_tx
:
tokio
::
sync
::
mpsc
::
Sender
<
ProcessedEndpoints
>
,
cancel
:
CancellationToken
,
...
...
lib/llm/src/kv_router/publisher.rs
View file @
602352ce
...
...
@@ -15,7 +15,7 @@
use
crate
::
kv_router
::{
indexer
::
RouterEvent
,
protocols
::
*
,
KV_EVENT_SUBJECT
};
use
async_trait
::
async_trait
;
use
dyn
e
mo_runtime
::{
use
dyn
a
mo_runtime
::{
component
::
Component
,
pipeline
::{
network
::
Ingress
,
AsyncEngine
,
AsyncEngineContextProvider
,
ManyOut
,
ResponseStream
,
...
...
lib/llm/src/kv_router/scheduler.rs
View file @
602352ce
...
...
@@ -13,8 +13,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use
dyn
e
mo_runtime
::
component
::
Namespace
;
use
dyn
e
mo_runtime
::
traits
::
events
::
EventPublisher
;
use
dyn
a
mo_runtime
::
component
::
Namespace
;
use
dyn
a
mo_runtime
::
traits
::
events
::
EventPublisher
;
use
serde
::{
Deserialize
,
Serialize
};
use
std
::
borrow
::
BorrowMut
;
use
std
::
cmp
::
min
;
...
...
lib/llm/src/kv_router/worker.rs
View file @
602352ce
...
...
@@ -19,12 +19,12 @@ pub use crate::kv_router::protocols::ForwardPassMetrics;
use
anyhow
::
Result
;
use
derive_builder
::
Builder
;
use
dyn
e
mo_runtime
::
pipeline
::
network
::{
use
dyn
a
mo_runtime
::
pipeline
::
network
::{
ingress
::
push_endpoint
::
PushEndpoint
,
PushWorkHandler
,
};
use
dyn
e
mo_runtime
::
transports
::
nats
::{
self
,
ServiceExt
};
use
dyn
a
mo_runtime
::
transports
::
nats
::{
self
,
ServiceExt
};
use
tokio
::
sync
::
watch
;
use
tokio_util
::
sync
::
CancellationToken
;
...
...
lib/llm/src/lib.rs
View file @
602352ce
...
...
@@ -13,9 +13,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! # Dyn
e
mo LLM
//! # Dyn
a
mo LLM
//!
//! The `dyn
e
mo
-
llm` crate is a Rust library that provides a set of traits and types for building
//! The `dyn
a
mo
.
llm` crate is a Rust library that provides a set of traits and types for building
//! distributed LLM inference solutions.
pub
mod
backend
;
...
...
lib/llm/src/model_card/model.rs
View file @
602352ce
...
...
@@ -37,7 +37,7 @@ use std::time::Duration;
use
derive_builder
::
Builder
;
use
dyn
e
mo_runtime
::
slug
::
Slug
;
use
dyn
a
mo_runtime
::
slug
::
Slug
;
pub
const
BUCKET_NAME
:
&
str
=
"mdc"
;
...
...
lib/llm/src/preprocessor.rs
View file @
602352ce
...
...
@@ -35,11 +35,11 @@ use tracing;
use
crate
::
model_card
::
model
::{
ModelDeploymentCard
,
ModelInfo
,
TokenizerKind
};
use
crate
::
preprocessor
::
prompt
::
OAIChatLikeRequest
;
use
dyn
e
mo_runtime
::
engine
::{
AsyncEngine
,
AsyncEngineContextProvider
,
ResponseStream
};
use
dyn
e
mo_runtime
::
pipeline
::{
use
dyn
a
mo_runtime
::
engine
::{
AsyncEngine
,
AsyncEngineContextProvider
,
ResponseStream
};
use
dyn
a
mo_runtime
::
pipeline
::{
async_trait
,
AsyncEngineContext
,
Error
,
ManyOut
,
Operator
,
SingleIn
,
};
use
dyn
e
mo_runtime
::
protocols
::
annotated
::{
Annotated
,
AnnotationsProvider
};
use
dyn
a
mo_runtime
::
protocols
::
annotated
::{
Annotated
,
AnnotationsProvider
};
use
crate
::
protocols
::{
common
::{
SamplingOptionsProvider
,
StopConditionsProvider
},
...
...
lib/llm/src/protocols.rs
View file @
602352ce
...
...
@@ -13,10 +13,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! # Dyn
e
mo LLM Protocols
//! # Dyn
a
mo LLM Protocols
//!
//! This module contains the protocols, i.e. messages formats, used to exchange requests and responses
//! both publicly via the HTTP API and internally between Dyn
e
mo components.
//! both publicly via the HTTP API and internally between Dyn
a
mo components.
//!
use
futures
::{
Stream
,
StreamExt
};
...
...
@@ -33,10 +33,10 @@ pub type DataStream<T> = Pin<Box<dyn Stream<Item = T> + Send + Sync>>;
// TODO: This is an awkward dependency that we need to address
// Originally, all the Annotated/SSE Codec bits where in the LLM protocol module; however, [Annotated]
// has become the common response envelope for dyn
e
mo
-distributed
.
// has become the common response envelope for dyn
a
mo.
// We may want to move the original Annotated back here and has a Infallible conversion to the the
// ResponseEnvelop in dyn
e
mo
-distributed
.
pub
use
dyn
e
mo_runtime
::
protocols
::
annotated
::
Annotated
;
// ResponseEnvelop in dyn
a
mo.
pub
use
dyn
a
mo_runtime
::
protocols
::
annotated
::
Annotated
;
/// The LLM responses have multiple different fields and nests of objects to get to the actual
/// text completion returned. This trait can be applied to the `choice` level objects to extract
...
...
lib/llm/src/protocols/common/preprocessor.rs
View file @
602352ce
...
...
@@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize};
use
super
::{
SamplingOptions
,
StopConditions
};
use
crate
::
protocols
::
TokenIdType
;
/// [`PreprocessedRequest`] is the internal representation of an LLM request. The [`dyn
e
mo
-
llm-preprocessor`]
/// [`PreprocessedRequest`] is the internal representation of an LLM request. The [`dyn
a
mo
.
llm-preprocessor`]
/// crate is responsible for converting request from the public APIs to this internal representation.
#[derive(Serialize,
Deserialize,
Debug,
Clone,
Builder)]
pub
struct
PreprocessedRequest
{
...
...
lib/llm/src/protocols/openai/chat_completions.rs
View file @
602352ce
...
...
@@ -17,7 +17,7 @@ use super::nvext::NvExt;
use
super
::
nvext
::
NvExtProvider
;
use
super
::
OpenAISamplingOptionsProvider
;
use
super
::
OpenAIStopConditionsProvider
;
use
dyn
e
mo_runtime
::
protocols
::
annotated
::
AnnotationsProvider
;
use
dyn
a
mo_runtime
::
protocols
::
annotated
::
AnnotationsProvider
;
use
serde
::{
Deserialize
,
Serialize
};
use
validator
::
Validate
;
...
...
lib/llm/src/protocols/openai/completions.rs
View file @
602352ce
...
...
@@ -31,7 +31,7 @@ use super::{
CompletionUsage
,
ContentProvider
,
OpenAISamplingOptionsProvider
,
OpenAIStopConditionsProvider
,
};
use
dyn
e
mo_runtime
::
protocols
::
annotated
::
AnnotationsProvider
;
use
dyn
a
mo_runtime
::
protocols
::
annotated
::
AnnotationsProvider
;
#[derive(Serialize,
Deserialize,
Validate,
Debug,
Clone)]
pub
struct
CompletionRequest
{
...
...
lib/llm/src/tokenizers/README.md
View file @
602352ce
...
...
@@ -13,7 +13,7 @@
#### HuggingFace Tokenizer
```
rust
use
dyn
e
mo_llm
::
tokenizers
::
hf
::
HuggingFaceTokenizer
;
use
dyn
a
mo_llm
::
tokenizers
::
hf
::
HuggingFaceTokenizer
;
let
hf_tokenizer
=
HuggingFaceTokenizer
::
from_file
(
"tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
)
.expect
(
"Failed to load HuggingFace tokenizer"
);
...
...
@@ -22,7 +22,7 @@ let hf_tokenizer = HuggingFaceTokenizer::from_file("tests/data/sample-models/Tin
### Encoding and Decoding Text
```
rust
use
dyn
e
mo_llm
::
tokenizers
::{
HuggingFaceTokenizer
,
traits
::{
Encoder
,
Decoder
}};
use
dyn
a
mo_llm
::
tokenizers
::{
HuggingFaceTokenizer
,
traits
::{
Encoder
,
Decoder
}};
let
tokenizer
=
HuggingFaceTokenizer
::
from_file
(
"tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
)
.expect
(
"Failed to load HuggingFace tokenizer"
);
...
...
@@ -40,7 +40,7 @@ assert_eq!(text, decoded_text);
// Using the Sequence object for encoding and decoding
use
dyn
e
mo_llm
::
tokenizers
::{
Sequence
,
Tokenizer
};
use
dyn
a
mo_llm
::
tokenizers
::{
Sequence
,
Tokenizer
};
use
std
::
sync
::{
Arc
,
RwLock
};
let
tokenizer
=
Tokenizer
::
from
(
Arc
::
new
(
tokenizer
));
...
...
lib/llm/src/types.rs
View file @
602352ce
...
...
@@ -19,7 +19,7 @@ pub use protocols::{Annotated, TokenIdType};
pub
mod
openai
{
use
super
::
*
;
use
dyn
e
mo_runtime
::
pipeline
::{
ServerStreamingEngine
,
UnaryEngine
};
use
dyn
a
mo_runtime
::
pipeline
::{
ServerStreamingEngine
,
UnaryEngine
};
pub
mod
completions
{
use
super
::
*
;
...
...
lib/llm/tests/aggregators.rs
View file @
602352ce
...
...
@@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use
dyn
e
mo_llm
::
protocols
::{
use
dyn
a
mo_llm
::
protocols
::{
codec
::{
create_message_stream
,
Message
,
SseCodecError
},
openai
::{
chat_completions
::
NvCreateChatCompletionResponse
,
completions
::
CompletionResponse
},
ContentProvider
,
DataStream
,
...
...
lib/llm/tests/backend.rs
View file @
602352ce
...
...
@@ -13,8 +13,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use
dyn
e
mo_llm
::
backend
::
Backend
;
use
dyn
e
mo_llm
::
model_card
::
model
::
ModelDeploymentCard
;
use
dyn
a
mo_llm
::
backend
::
Backend
;
use
dyn
a
mo_llm
::
model_card
::
model
::
ModelDeploymentCard
;
#[tokio::test]
async
fn
test_sequence_factory
()
{
...
...
lib/llm/tests/http-service.rs
View file @
602352ce
...
...
@@ -15,20 +15,20 @@
use
anyhow
::
Error
;
use
async_stream
::
stream
;
use
dyn
e
mo_llm
::
http
::
service
::{
use
dyn
a
mo_llm
::
http
::
service
::{
error
::
HttpError
,
metrics
::{
Endpoint
,
RequestType
,
Status
},
service_v2
::
HttpService
,
Metrics
,
};
use
dyn
e
mo_llm
::
protocols
::{
use
dyn
a
mo_llm
::
protocols
::{
openai
::{
chat_completions
::{
NvCreateChatCompletionRequest
,
NvCreateChatCompletionStreamResponse
},
completions
::{
CompletionRequest
,
CompletionResponse
},
},
Annotated
,
};
use
dyn
e
mo_runtime
::{
use
dyn
a
mo_runtime
::{
pipeline
::{
async_trait
,
AsyncEngine
,
AsyncEngineContextProvider
,
ManyOut
,
ResponseStream
,
SingleIn
,
},
...
...
lib/llm/tests/model_card.rs
View file @
602352ce
...
...
@@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use
dyn
e
mo_llm
::
model_card
::
model
::{
ModelDeploymentCard
,
PromptFormatterArtifact
,
TokenizerKind
};
use
dyn
a
mo_llm
::
model_card
::
model
::{
ModelDeploymentCard
,
PromptFormatterArtifact
,
TokenizerKind
};
use
tempfile
::
tempdir
;
const
HF_PATH
:
&
str
=
"tests/data/sample-models/TinyLlama_v1.1"
;
...
...
lib/llm/tests/openai_completions.rs
View file @
602352ce
...
...
@@ -14,7 +14,7 @@
// limitations under the License.
use
async_openai
::
types
::
CreateCompletionRequestArgs
;
use
dyn
e
mo_llm
::
protocols
::
openai
::{
self
,
completions
::
CompletionRequest
};
use
dyn
a
mo_llm
::
protocols
::
openai
::{
self
,
completions
::
CompletionRequest
};
use
serde
::{
Deserialize
,
Serialize
};
#[derive(Serialize,
Deserialize,
Debug,
Clone)]
...
...
Prev
1
…
16
17
18
19
20
21
22
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment