Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OpenDAS
dynamo
Commits
602352ce
Commit
602352ce
authored
Mar 08, 2025
by
Neelay Shah
Committed by
GitHub
Mar 08, 2025
Browse files
chore: rename dynamo (#44)
Co-authored-by:
Biswa Panda
<
biswa.panda@gmail.com
>
parent
ecf53ce2
Changes
431
Show whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
39 additions
and
39 deletions
+39
-39
lib/llm/src/http/service/openai.rs
lib/llm/src/http/service/openai.rs
+3
-3
lib/llm/src/kv_router.rs
lib/llm/src/kv_router.rs
+3
-3
lib/llm/src/kv_router/metrics_aggregator.rs
lib/llm/src/kv_router/metrics_aggregator.rs
+2
-2
lib/llm/src/kv_router/publisher.rs
lib/llm/src/kv_router/publisher.rs
+1
-1
lib/llm/src/kv_router/scheduler.rs
lib/llm/src/kv_router/scheduler.rs
+2
-2
lib/llm/src/kv_router/worker.rs
lib/llm/src/kv_router/worker.rs
+2
-2
lib/llm/src/lib.rs
lib/llm/src/lib.rs
+2
-2
lib/llm/src/model_card/model.rs
lib/llm/src/model_card/model.rs
+1
-1
lib/llm/src/preprocessor.rs
lib/llm/src/preprocessor.rs
+3
-3
lib/llm/src/protocols.rs
lib/llm/src/protocols.rs
+5
-5
lib/llm/src/protocols/common/preprocessor.rs
lib/llm/src/protocols/common/preprocessor.rs
+1
-1
lib/llm/src/protocols/openai/chat_completions.rs
lib/llm/src/protocols/openai/chat_completions.rs
+1
-1
lib/llm/src/protocols/openai/completions.rs
lib/llm/src/protocols/openai/completions.rs
+1
-1
lib/llm/src/tokenizers/README.md
lib/llm/src/tokenizers/README.md
+3
-3
lib/llm/src/types.rs
lib/llm/src/types.rs
+1
-1
lib/llm/tests/aggregators.rs
lib/llm/tests/aggregators.rs
+1
-1
lib/llm/tests/backend.rs
lib/llm/tests/backend.rs
+2
-2
lib/llm/tests/http-service.rs
lib/llm/tests/http-service.rs
+3
-3
lib/llm/tests/model_card.rs
lib/llm/tests/model_card.rs
+1
-1
lib/llm/tests/openai_completions.rs
lib/llm/tests/openai_completions.rs
+1
-1
No files found.
lib/llm/src/http/service/openai.rs
View file @
602352ce
...
...
@@ -48,7 +48,7 @@ use crate::types::{
Annotated
,
};
use
dyn
e
mo_runtime
::
pipeline
::{
AsyncEngineContext
,
Context
};
use
dyn
a
mo_runtime
::
pipeline
::{
AsyncEngineContext
,
Context
};
#[derive(Serialize,
Deserialize)]
pub
(
crate
)
struct
ErrorResponse
{
...
...
@@ -91,7 +91,7 @@ impl ErrorResponse {
)
}
/// The OAI endpoints call an [`dyn
e
mo
_
runtime::engine::AsyncEngine`] which are specialized to return
/// The OAI endpoints call an [`dyn
a
mo
.
runtime::engine::AsyncEngine`] which are specialized to return
/// an [`anyhow::Error`]. This method will convert the [`anyhow::Error`] into an [`HttpError`].
/// If successful, it will return the [`HttpError`] as an [`ErrorResponse::internal_server_error`]
/// with the details of the error.
...
...
@@ -516,7 +516,7 @@ pub fn list_models_router(
path
:
Option
<
String
>
,
)
->
(
Vec
<
RouteDoc
>
,
Router
)
{
// TODO: Why do we have this endpoint?
let
custom_path
=
path
.unwrap_or
(
"/dyn
e
mo/alpha/list-models"
.to_string
());
let
custom_path
=
path
.unwrap_or
(
"/dyn
a
mo/alpha/list-models"
.to_string
());
let
doc_for_custom
=
RouteDoc
::
new
(
axum
::
http
::
Method
::
GET
,
&
custom_path
);
// Standard OpenAI compatible list models endpoint
...
...
lib/llm/src/kv_router.rs
View file @
602352ce
...
...
@@ -14,7 +14,7 @@
// limitations under the License.
use
anyhow
::
Result
;
use
dyn
e
mo_runtime
::{
component
::
Component
,
component
::
Namespace
,
DistributedRuntime
};
use
dyn
a
mo_runtime
::{
component
::
Component
,
component
::
Namespace
,
DistributedRuntime
};
use
futures
::
stream
::
StreamExt
;
use
std
::{
sync
::
Arc
,
time
::
Duration
};
use
tokio_util
::
sync
::
CancellationToken
;
...
...
@@ -67,7 +67,7 @@ impl KvRouter {
}
pub
async
fn
new
(
nats_client
:
dyn
e
mo_runtime
::
transports
::
nats
::
Client
,
nats_client
:
dyn
a
mo_runtime
::
transports
::
nats
::
Client
,
service_name
:
String
,
kv_subject
:
String
,
namespace
:
Namespace
,
...
...
@@ -141,7 +141,7 @@ impl KvRouter {
}
async
fn
collect_endpoints
(
nats_client
:
dyn
e
mo_runtime
::
transports
::
nats
::
Client
,
nats_client
:
dyn
a
mo_runtime
::
transports
::
nats
::
Client
,
service_name
:
String
,
ep_tx
:
tokio
::
sync
::
mpsc
::
Sender
<
ProcessedEndpoints
>
,
cancel
:
CancellationToken
,
...
...
lib/llm/src/kv_router/metrics_aggregator.rs
View file @
602352ce
...
...
@@ -19,7 +19,7 @@ pub use crate::kv_router::protocols::ForwardPassMetrics;
use
crate
::
kv_router
::
scheduler
::{
Endpoint
,
Service
};
use
crate
::
kv_router
::
ProcessedEndpoints
;
use
dyn
e
mo_runtime
::
component
::
Component
;
use
dyn
a
mo_runtime
::
component
::
Component
;
use
std
::
time
::
Duration
;
use
tokio_util
::
sync
::
CancellationToken
;
...
...
@@ -81,7 +81,7 @@ impl KvMetricsAggregator {
}
async
fn
collect_endpoints
(
nats_client
:
dyn
e
mo_runtime
::
transports
::
nats
::
Client
,
nats_client
:
dyn
a
mo_runtime
::
transports
::
nats
::
Client
,
service_name
:
String
,
ep_tx
:
tokio
::
sync
::
mpsc
::
Sender
<
ProcessedEndpoints
>
,
cancel
:
CancellationToken
,
...
...
lib/llm/src/kv_router/publisher.rs
View file @
602352ce
...
...
@@ -15,7 +15,7 @@
use
crate
::
kv_router
::{
indexer
::
RouterEvent
,
protocols
::
*
,
KV_EVENT_SUBJECT
};
use
async_trait
::
async_trait
;
use
dyn
e
mo_runtime
::{
use
dyn
a
mo_runtime
::{
component
::
Component
,
pipeline
::{
network
::
Ingress
,
AsyncEngine
,
AsyncEngineContextProvider
,
ManyOut
,
ResponseStream
,
...
...
lib/llm/src/kv_router/scheduler.rs
View file @
602352ce
...
...
@@ -13,8 +13,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use
dyn
e
mo_runtime
::
component
::
Namespace
;
use
dyn
e
mo_runtime
::
traits
::
events
::
EventPublisher
;
use
dyn
a
mo_runtime
::
component
::
Namespace
;
use
dyn
a
mo_runtime
::
traits
::
events
::
EventPublisher
;
use
serde
::{
Deserialize
,
Serialize
};
use
std
::
borrow
::
BorrowMut
;
use
std
::
cmp
::
min
;
...
...
lib/llm/src/kv_router/worker.rs
View file @
602352ce
...
...
@@ -19,12 +19,12 @@ pub use crate::kv_router::protocols::ForwardPassMetrics;
use
anyhow
::
Result
;
use
derive_builder
::
Builder
;
use
dyn
e
mo_runtime
::
pipeline
::
network
::{
use
dyn
a
mo_runtime
::
pipeline
::
network
::{
ingress
::
push_endpoint
::
PushEndpoint
,
PushWorkHandler
,
};
use
dyn
e
mo_runtime
::
transports
::
nats
::{
self
,
ServiceExt
};
use
dyn
a
mo_runtime
::
transports
::
nats
::{
self
,
ServiceExt
};
use
tokio
::
sync
::
watch
;
use
tokio_util
::
sync
::
CancellationToken
;
...
...
lib/llm/src/lib.rs
View file @
602352ce
...
...
@@ -13,9 +13,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! # Dyn
e
mo LLM
//! # Dyn
a
mo LLM
//!
//! The `dyn
e
mo
-
llm` crate is a Rust library that provides a set of traits and types for building
//! The `dyn
a
mo
.
llm` crate is a Rust library that provides a set of traits and types for building
//! distributed LLM inference solutions.
pub
mod
backend
;
...
...
lib/llm/src/model_card/model.rs
View file @
602352ce
...
...
@@ -37,7 +37,7 @@ use std::time::Duration;
use
derive_builder
::
Builder
;
use
dyn
e
mo_runtime
::
slug
::
Slug
;
use
dyn
a
mo_runtime
::
slug
::
Slug
;
pub
const
BUCKET_NAME
:
&
str
=
"mdc"
;
...
...
lib/llm/src/preprocessor.rs
View file @
602352ce
...
...
@@ -35,11 +35,11 @@ use tracing;
use
crate
::
model_card
::
model
::{
ModelDeploymentCard
,
ModelInfo
,
TokenizerKind
};
use
crate
::
preprocessor
::
prompt
::
OAIChatLikeRequest
;
use
dyn
e
mo_runtime
::
engine
::{
AsyncEngine
,
AsyncEngineContextProvider
,
ResponseStream
};
use
dyn
e
mo_runtime
::
pipeline
::{
use
dyn
a
mo_runtime
::
engine
::{
AsyncEngine
,
AsyncEngineContextProvider
,
ResponseStream
};
use
dyn
a
mo_runtime
::
pipeline
::{
async_trait
,
AsyncEngineContext
,
Error
,
ManyOut
,
Operator
,
SingleIn
,
};
use
dyn
e
mo_runtime
::
protocols
::
annotated
::{
Annotated
,
AnnotationsProvider
};
use
dyn
a
mo_runtime
::
protocols
::
annotated
::{
Annotated
,
AnnotationsProvider
};
use
crate
::
protocols
::{
common
::{
SamplingOptionsProvider
,
StopConditionsProvider
},
...
...
lib/llm/src/protocols.rs
View file @
602352ce
...
...
@@ -13,10 +13,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! # Dyn
e
mo LLM Protocols
//! # Dyn
a
mo LLM Protocols
//!
//! This module contains the protocols, i.e. messages formats, used to exchange requests and responses
//! both publicly via the HTTP API and internally between Dyn
e
mo components.
//! both publicly via the HTTP API and internally between Dyn
a
mo components.
//!
use
futures
::{
Stream
,
StreamExt
};
...
...
@@ -33,10 +33,10 @@ pub type DataStream<T> = Pin<Box<dyn Stream<Item = T> + Send + Sync>>;
// TODO: This is an awkward dependency that we need to address
// Originally, all the Annotated/SSE Codec bits where in the LLM protocol module; however, [Annotated]
// has become the common response envelope for dyn
e
mo
-distributed
.
// has become the common response envelope for dyn
a
mo.
// We may want to move the original Annotated back here and has a Infallible conversion to the the
// ResponseEnvelop in dyn
e
mo
-distributed
.
pub
use
dyn
e
mo_runtime
::
protocols
::
annotated
::
Annotated
;
// ResponseEnvelop in dyn
a
mo.
pub
use
dyn
a
mo_runtime
::
protocols
::
annotated
::
Annotated
;
/// The LLM responses have multiple different fields and nests of objects to get to the actual
/// text completion returned. This trait can be applied to the `choice` level objects to extract
...
...
lib/llm/src/protocols/common/preprocessor.rs
View file @
602352ce
...
...
@@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize};
use
super
::{
SamplingOptions
,
StopConditions
};
use
crate
::
protocols
::
TokenIdType
;
/// [`PreprocessedRequest`] is the internal representation of an LLM request. The [`dyn
e
mo
-
llm-preprocessor`]
/// [`PreprocessedRequest`] is the internal representation of an LLM request. The [`dyn
a
mo
.
llm-preprocessor`]
/// crate is responsible for converting request from the public APIs to this internal representation.
#[derive(Serialize,
Deserialize,
Debug,
Clone,
Builder)]
pub
struct
PreprocessedRequest
{
...
...
lib/llm/src/protocols/openai/chat_completions.rs
View file @
602352ce
...
...
@@ -17,7 +17,7 @@ use super::nvext::NvExt;
use
super
::
nvext
::
NvExtProvider
;
use
super
::
OpenAISamplingOptionsProvider
;
use
super
::
OpenAIStopConditionsProvider
;
use
dyn
e
mo_runtime
::
protocols
::
annotated
::
AnnotationsProvider
;
use
dyn
a
mo_runtime
::
protocols
::
annotated
::
AnnotationsProvider
;
use
serde
::{
Deserialize
,
Serialize
};
use
validator
::
Validate
;
...
...
lib/llm/src/protocols/openai/completions.rs
View file @
602352ce
...
...
@@ -31,7 +31,7 @@ use super::{
CompletionUsage
,
ContentProvider
,
OpenAISamplingOptionsProvider
,
OpenAIStopConditionsProvider
,
};
use
dyn
e
mo_runtime
::
protocols
::
annotated
::
AnnotationsProvider
;
use
dyn
a
mo_runtime
::
protocols
::
annotated
::
AnnotationsProvider
;
#[derive(Serialize,
Deserialize,
Validate,
Debug,
Clone)]
pub
struct
CompletionRequest
{
...
...
lib/llm/src/tokenizers/README.md
View file @
602352ce
...
...
@@ -13,7 +13,7 @@
#### HuggingFace Tokenizer
```
rust
use
dyn
e
mo_llm
::
tokenizers
::
hf
::
HuggingFaceTokenizer
;
use
dyn
a
mo_llm
::
tokenizers
::
hf
::
HuggingFaceTokenizer
;
let
hf_tokenizer
=
HuggingFaceTokenizer
::
from_file
(
"tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
)
.expect
(
"Failed to load HuggingFace tokenizer"
);
...
...
@@ -22,7 +22,7 @@ let hf_tokenizer = HuggingFaceTokenizer::from_file("tests/data/sample-models/Tin
### Encoding and Decoding Text
```
rust
use
dyn
e
mo_llm
::
tokenizers
::{
HuggingFaceTokenizer
,
traits
::{
Encoder
,
Decoder
}};
use
dyn
a
mo_llm
::
tokenizers
::{
HuggingFaceTokenizer
,
traits
::{
Encoder
,
Decoder
}};
let
tokenizer
=
HuggingFaceTokenizer
::
from_file
(
"tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
)
.expect
(
"Failed to load HuggingFace tokenizer"
);
...
...
@@ -40,7 +40,7 @@ assert_eq!(text, decoded_text);
// Using the Sequence object for encoding and decoding
use
dyn
e
mo_llm
::
tokenizers
::{
Sequence
,
Tokenizer
};
use
dyn
a
mo_llm
::
tokenizers
::{
Sequence
,
Tokenizer
};
use
std
::
sync
::{
Arc
,
RwLock
};
let
tokenizer
=
Tokenizer
::
from
(
Arc
::
new
(
tokenizer
));
...
...
lib/llm/src/types.rs
View file @
602352ce
...
...
@@ -19,7 +19,7 @@ pub use protocols::{Annotated, TokenIdType};
pub
mod
openai
{
use
super
::
*
;
use
dyn
e
mo_runtime
::
pipeline
::{
ServerStreamingEngine
,
UnaryEngine
};
use
dyn
a
mo_runtime
::
pipeline
::{
ServerStreamingEngine
,
UnaryEngine
};
pub
mod
completions
{
use
super
::
*
;
...
...
lib/llm/tests/aggregators.rs
View file @
602352ce
...
...
@@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use
dyn
e
mo_llm
::
protocols
::{
use
dyn
a
mo_llm
::
protocols
::{
codec
::{
create_message_stream
,
Message
,
SseCodecError
},
openai
::{
chat_completions
::
NvCreateChatCompletionResponse
,
completions
::
CompletionResponse
},
ContentProvider
,
DataStream
,
...
...
lib/llm/tests/backend.rs
View file @
602352ce
...
...
@@ -13,8 +13,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use
dyn
e
mo_llm
::
backend
::
Backend
;
use
dyn
e
mo_llm
::
model_card
::
model
::
ModelDeploymentCard
;
use
dyn
a
mo_llm
::
backend
::
Backend
;
use
dyn
a
mo_llm
::
model_card
::
model
::
ModelDeploymentCard
;
#[tokio::test]
async
fn
test_sequence_factory
()
{
...
...
lib/llm/tests/http-service.rs
View file @
602352ce
...
...
@@ -15,20 +15,20 @@
use
anyhow
::
Error
;
use
async_stream
::
stream
;
use
dyn
e
mo_llm
::
http
::
service
::{
use
dyn
a
mo_llm
::
http
::
service
::{
error
::
HttpError
,
metrics
::{
Endpoint
,
RequestType
,
Status
},
service_v2
::
HttpService
,
Metrics
,
};
use
dyn
e
mo_llm
::
protocols
::{
use
dyn
a
mo_llm
::
protocols
::{
openai
::{
chat_completions
::{
NvCreateChatCompletionRequest
,
NvCreateChatCompletionStreamResponse
},
completions
::{
CompletionRequest
,
CompletionResponse
},
},
Annotated
,
};
use
dyn
e
mo_runtime
::{
use
dyn
a
mo_runtime
::{
pipeline
::{
async_trait
,
AsyncEngine
,
AsyncEngineContextProvider
,
ManyOut
,
ResponseStream
,
SingleIn
,
},
...
...
lib/llm/tests/model_card.rs
View file @
602352ce
...
...
@@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use
dyn
e
mo_llm
::
model_card
::
model
::{
ModelDeploymentCard
,
PromptFormatterArtifact
,
TokenizerKind
};
use
dyn
a
mo_llm
::
model_card
::
model
::{
ModelDeploymentCard
,
PromptFormatterArtifact
,
TokenizerKind
};
use
tempfile
::
tempdir
;
const
HF_PATH
:
&
str
=
"tests/data/sample-models/TinyLlama_v1.1"
;
...
...
lib/llm/tests/openai_completions.rs
View file @
602352ce
...
...
@@ -14,7 +14,7 @@
// limitations under the License.
use
async_openai
::
types
::
CreateCompletionRequestArgs
;
use
dyn
e
mo_llm
::
protocols
::
openai
::{
self
,
completions
::
CompletionRequest
};
use
dyn
a
mo_llm
::
protocols
::
openai
::{
self
,
completions
::
CompletionRequest
};
use
serde
::{
Deserialize
,
Serialize
};
#[derive(Serialize,
Deserialize,
Debug,
Clone)]
...
...
Prev
1
…
16
17
18
19
20
21
22
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment