Unverified Commit fcb91e4b authored by Graham King's avatar Graham King Committed by GitHub
Browse files

refactor(storage): Remove the stuttering from key_value_store. (#4604)


Signed-off-by: default avatarGraham King <grahamk@nvidia.com>
parent 7a384793
...@@ -6,7 +6,7 @@ use dynamo_llm::entrypoint::EngineConfig; ...@@ -6,7 +6,7 @@ use dynamo_llm::entrypoint::EngineConfig;
use dynamo_llm::entrypoint::input::Input; use dynamo_llm::entrypoint::input::Input;
use dynamo_llm::local_model::{LocalModel, LocalModelBuilder}; use dynamo_llm::local_model::{LocalModel, LocalModelBuilder};
use dynamo_runtime::distributed::{DistributedConfig, RequestPlaneMode}; use dynamo_runtime::distributed::{DistributedConfig, RequestPlaneMode};
use dynamo_runtime::storage::key_value_store::KeyValueStoreSelect; use dynamo_runtime::storage::kv;
use dynamo_runtime::transports::nats; use dynamo_runtime::transports::nats;
use dynamo_runtime::{DistributedRuntime, Runtime}; use dynamo_runtime::{DistributedRuntime, Runtime};
...@@ -82,7 +82,7 @@ pub async fn run( ...@@ -82,7 +82,7 @@ pub async fn run(
DistributedConfig::process_local() DistributedConfig::process_local()
} else { } else {
// Normal case // Normal case
let selected_store: KeyValueStoreSelect = flags.store_kv.parse()?; let selected_store: kv::Selector = flags.store_kv.parse()?;
let request_plane: RequestPlaneMode = flags.request_plane.parse()?; let request_plane: RequestPlaneMode = flags.request_plane.parse()?;
DistributedConfig { DistributedConfig {
store_backend: selected_store, store_backend: selected_store,
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
use dynamo_llm::local_model::LocalModel; use dynamo_llm::local_model::LocalModel;
use dynamo_runtime::distributed::{DistributedConfig, RequestPlaneMode}; use dynamo_runtime::distributed::{DistributedConfig, RequestPlaneMode};
use dynamo_runtime::storage::key_value_store::KeyValueStoreSelect; use dynamo_runtime::storage::kv;
use futures::StreamExt; use futures::StreamExt;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use pyo3::IntoPyObjectExt; use pyo3::IntoPyObjectExt;
...@@ -455,7 +455,7 @@ enum ModelInput { ...@@ -455,7 +455,7 @@ enum ModelInput {
impl DistributedRuntime { impl DistributedRuntime {
#[new] #[new]
fn new(event_loop: PyObject, store_kv: String, request_plane: String) -> PyResult<Self> { fn new(event_loop: PyObject, store_kv: String, request_plane: String) -> PyResult<Self> {
let selected_kv_store: KeyValueStoreSelect = store_kv.parse().map_err(to_pyerr)?; let selected_kv_store: kv::Selector = store_kv.parse().map_err(to_pyerr)?;
let request_plane: RequestPlaneMode = request_plane.parse().map_err(to_pyerr)?; let request_plane: RequestPlaneMode = request_plane.parse().map_err(to_pyerr)?;
// Try to get existing runtime first, create new Worker only if needed // Try to get existing runtime first, create new Worker only if needed
......
...@@ -21,7 +21,7 @@ use derive_builder::Builder; ...@@ -21,7 +21,7 @@ use derive_builder::Builder;
use dynamo_runtime::discovery::{Discovery, KVStoreDiscovery}; use dynamo_runtime::discovery::{Discovery, KVStoreDiscovery};
use dynamo_runtime::logging::make_request_span; use dynamo_runtime::logging::make_request_span;
use dynamo_runtime::metrics::prometheus_names::name_prefix; use dynamo_runtime::metrics::prometheus_names::name_prefix;
use dynamo_runtime::storage::key_value_store::KeyValueStoreManager; use dynamo_runtime::storage::kv;
use std::net::SocketAddr; use std::net::SocketAddr;
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
...@@ -31,7 +31,7 @@ use tower_http::trace::TraceLayer; ...@@ -31,7 +31,7 @@ use tower_http::trace::TraceLayer;
pub struct State { pub struct State {
metrics: Arc<Metrics>, metrics: Arc<Metrics>,
manager: Arc<ModelManager>, manager: Arc<ModelManager>,
store: KeyValueStoreManager, store: kv::Manager,
discovery_client: Arc<dyn Discovery>, discovery_client: Arc<dyn Discovery>,
flags: StateFlags, flags: StateFlags,
} }
...@@ -73,7 +73,7 @@ impl StateFlags { ...@@ -73,7 +73,7 @@ impl StateFlags {
} }
impl State { impl State {
pub fn new(manager: Arc<ModelManager>, store: KeyValueStoreManager) -> Self { pub fn new(manager: Arc<ModelManager>, store: kv::Manager) -> Self {
// Initialize discovery backed by KV store // Initialize discovery backed by KV store
// Create a cancellation token for the discovery's watch streams // Create a cancellation token for the discovery's watch streams
let discovery_client = { let discovery_client = {
...@@ -108,7 +108,7 @@ impl State { ...@@ -108,7 +108,7 @@ impl State {
self.manager.clone() self.manager.clone()
} }
pub fn store(&self) -> &KeyValueStoreManager { pub fn store(&self) -> &kv::Manager {
&self.store &self.store
} }
...@@ -178,7 +178,7 @@ pub struct HttpServiceConfig { ...@@ -178,7 +178,7 @@ pub struct HttpServiceConfig {
request_template: Option<RequestTemplate>, request_template: Option<RequestTemplate>,
#[builder(default)] #[builder(default)]
store: KeyValueStoreManager, store: kv::Manager,
// DEPRECATED: To be removed after custom backends migrate to Dynamo backend. // DEPRECATED: To be removed after custom backends migrate to Dynamo backend.
#[builder(default = "None")] #[builder(default = "None")]
......
...@@ -21,7 +21,7 @@ use crate::local_model::runtime_config::ModelRuntimeConfig; ...@@ -21,7 +21,7 @@ use crate::local_model::runtime_config::ModelRuntimeConfig;
use crate::model_type::{ModelInput, ModelType}; use crate::model_type::{ModelInput, ModelType};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use derive_builder::Builder; use derive_builder::Builder;
use dynamo_runtime::{slug::Slug, storage::key_value_store::Versioned}; use dynamo_runtime::{slug::Slug, storage::kv};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokenizers::Tokenizer as HfTokenizer; use tokenizers::Tokenizer as HfTokenizer;
...@@ -543,7 +543,7 @@ impl PartialEq for ModelDeploymentCard { ...@@ -543,7 +543,7 @@ impl PartialEq for ModelDeploymentCard {
} }
/// A ModelDeploymentCard is published a single time per instance and never updated. /// A ModelDeploymentCard is published a single time per instance and never updated.
impl Versioned for ModelDeploymentCard { impl kv::Versioned for ModelDeploymentCard {
fn revision(&self) -> u64 { fn revision(&self) -> u64 {
0 0
} }
......
...@@ -17,7 +17,6 @@ use crate::{ ...@@ -17,7 +17,6 @@ use crate::{
AddressedPushRouter, AddressedRequest, AsyncEngine, Data, ManyOut, PushRouter, RouterMode, AddressedPushRouter, AddressedRequest, AsyncEngine, Data, ManyOut, PushRouter, RouterMode,
SingleIn, SingleIn,
}, },
storage::key_value_store::{KeyValueStoreManager, WatchEvent},
traits::DistributedRuntimeProvider, traits::DistributedRuntimeProvider,
transports::etcd::Client as EtcdClient, transports::etcd::Client as EtcdClient,
}; };
......
...@@ -15,7 +15,6 @@ use crate::{ ...@@ -15,7 +15,6 @@ use crate::{
distributed::RequestPlaneMode, distributed::RequestPlaneMode,
pipeline::network::{PushWorkHandler, ingress::push_endpoint::PushEndpoint}, pipeline::network::{PushWorkHandler, ingress::push_endpoint::PushEndpoint},
protocols::EndpointId, protocols::EndpointId,
storage::key_value_store,
traits::DistributedRuntimeProvider, traits::DistributedRuntimeProvider,
transports::nats, transports::nats,
}; };
......
...@@ -12,19 +12,19 @@ use tokio_util::sync::CancellationToken; ...@@ -12,19 +12,19 @@ use tokio_util::sync::CancellationToken;
use super::{ use super::{
Discovery, DiscoveryEvent, DiscoveryInstance, DiscoveryQuery, DiscoverySpec, DiscoveryStream, Discovery, DiscoveryEvent, DiscoveryInstance, DiscoveryQuery, DiscoverySpec, DiscoveryStream,
}; };
use crate::storage::key_value_store::{KeyValueStoreManager, WatchEvent}; use crate::storage::kv;
const INSTANCES_BUCKET: &str = "v1/instances"; const INSTANCES_BUCKET: &str = "v1/instances";
const MODELS_BUCKET: &str = "v1/mdc"; const MODELS_BUCKET: &str = "v1/mdc";
/// Discovery implementation backed by a KeyValueStore /// Discovery implementation backed by a kv::Store
pub struct KVStoreDiscovery { pub struct KVStoreDiscovery {
store: Arc<KeyValueStoreManager>, store: Arc<kv::Manager>,
cancel_token: CancellationToken, cancel_token: CancellationToken,
} }
impl KVStoreDiscovery { impl KVStoreDiscovery {
pub fn new(store: KeyValueStoreManager, cancel_token: CancellationToken) -> Self { pub fn new(store: kv::Manager, cancel_token: CancellationToken) -> Self {
Self { Self {
store: Arc::new(store), store: Arc::new(store),
cancel_token, cancel_token,
...@@ -184,7 +184,7 @@ impl Discovery for KVStoreDiscovery { ...@@ -184,7 +184,7 @@ impl Discovery for KVStoreDiscovery {
key_path key_path
); );
let bucket = self.store.get_or_create_bucket(bucket_name, None).await?; let bucket = self.store.get_or_create_bucket(bucket_name, None).await?;
let key = crate::storage::key_value_store::Key::new(key_path.clone()); let key = kv::Key::new(key_path.clone());
tracing::debug!( tracing::debug!(
"KVStoreDiscovery::register: Inserting into bucket={}, key={}", "KVStoreDiscovery::register: Inserting into bucket={}, key={}",
...@@ -251,7 +251,7 @@ impl Discovery for KVStoreDiscovery { ...@@ -251,7 +251,7 @@ impl Discovery for KVStoreDiscovery {
return Ok(()); return Ok(());
}; };
let key = crate::storage::key_value_store::Key::new(key_path.clone()); let key = kv::Key::new(key_path.clone());
// Delete the entry from the bucket // Delete the entry from the bucket
bucket.delete(&key).await?; bucket.delete(&key).await?;
...@@ -313,7 +313,7 @@ impl Discovery for KVStoreDiscovery { ...@@ -313,7 +313,7 @@ impl Discovery for KVStoreDiscovery {
// Use the provided cancellation token, or fall back to the default token // Use the provided cancellation token, or fall back to the default token
let cancel_token = cancel_token.unwrap_or_else(|| self.cancel_token.clone()); let cancel_token = cancel_token.unwrap_or_else(|| self.cancel_token.clone());
// Use the KeyValueStoreManager's watch mechanism // Use the kv::Manager's watch mechanism
let (_, mut rx) = self.store.clone().watch( let (_, mut rx) = self.store.clone().watch(
bucket_name, bucket_name,
None, // No TTL None, // No TTL
...@@ -324,7 +324,7 @@ impl Discovery for KVStoreDiscovery { ...@@ -324,7 +324,7 @@ impl Discovery for KVStoreDiscovery {
let stream = async_stream::stream! { let stream = async_stream::stream! {
while let Some(event) = rx.recv().await { while let Some(event) = rx.recv().await {
let discovery_event = match event { let discovery_event = match event {
WatchEvent::Put(kv) => { kv::WatchEvent::Put(kv) => {
// Check if this key matches our prefix // Check if this key matches our prefix
if !Self::matches_prefix(kv.key_str(), &prefix, bucket_name) { if !Self::matches_prefix(kv.key_str(), &prefix, bucket_name) {
continue; continue;
...@@ -344,7 +344,7 @@ impl Discovery for KVStoreDiscovery { ...@@ -344,7 +344,7 @@ impl Discovery for KVStoreDiscovery {
} }
} }
} }
WatchEvent::Delete(kv) => { kv::WatchEvent::Delete(kv) => {
let key_str = kv.as_ref(); let key_str = kv.as_ref();
// Check if this key matches our prefix // Check if this key matches our prefix
if !Self::matches_prefix(key_str, &prefix, bucket_name) { if !Self::matches_prefix(key_str, &prefix, bucket_name) {
...@@ -398,7 +398,7 @@ mod tests { ...@@ -398,7 +398,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_kv_store_discovery_register_endpoint() { async fn test_kv_store_discovery_register_endpoint() {
let store = KeyValueStoreManager::memory(); let store = kv::Manager::memory();
let cancel_token = CancellationToken::new(); let cancel_token = CancellationToken::new();
let client = KVStoreDiscovery::new(store, cancel_token); let client = KVStoreDiscovery::new(store, cancel_token);
...@@ -423,7 +423,7 @@ mod tests { ...@@ -423,7 +423,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_kv_store_discovery_list() { async fn test_kv_store_discovery_list() {
let store = KeyValueStoreManager::memory(); let store = kv::Manager::memory();
let cancel_token = CancellationToken::new(); let cancel_token = CancellationToken::new();
let client = KVStoreDiscovery::new(store, cancel_token); let client = KVStoreDiscovery::new(store, cancel_token);
...@@ -478,7 +478,7 @@ mod tests { ...@@ -478,7 +478,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_kv_store_discovery_watch() { async fn test_kv_store_discovery_watch() {
let store = KeyValueStoreManager::memory(); let store = kv::Manager::memory();
let cancel_token = CancellationToken::new(); let cancel_token = CancellationToken::new();
let client = Arc::new(KVStoreDiscovery::new(store, cancel_token.clone())); let client = Arc::new(KVStoreDiscovery::new(store, cancel_token.clone()));
......
...@@ -5,10 +5,7 @@ use crate::component::{Component, Instance}; ...@@ -5,10 +5,7 @@ use crate::component::{Component, Instance};
use crate::pipeline::PipelineError; use crate::pipeline::PipelineError;
use crate::pipeline::network::manager::NetworkManager; use crate::pipeline::network::manager::NetworkManager;
use crate::service::{ComponentNatsServerPrometheusMetrics, ServiceClient, ServiceSet}; use crate::service::{ComponentNatsServerPrometheusMetrics, ServiceClient, ServiceSet};
use crate::storage::key_value_store::{ use crate::storage::kv::{self, Store as _};
EtcdStore, KeyValueStore, KeyValueStoreEnum, KeyValueStoreManager, KeyValueStoreSelect,
MemoryStore,
};
use crate::transports::nats::DRTNatsClientPrometheusMetrics; use crate::transports::nats::DRTNatsClientPrometheusMetrics;
use crate::{ use crate::{
component::{self, ComponentBuilder, Endpoint, Namespace}, component::{self, ComponentBuilder, Endpoint, Namespace},
...@@ -48,7 +45,7 @@ pub struct DistributedRuntime { ...@@ -48,7 +45,7 @@ pub struct DistributedRuntime {
runtime: Runtime, runtime: Runtime,
nats_client: Option<transports::nats::Client>, nats_client: Option<transports::nats::Client>,
store: KeyValueStoreManager, store: kv::Manager,
network_manager: Arc<NetworkManager>, network_manager: Arc<NetworkManager>,
tcp_server: Arc<OnceCell<Arc<transports::tcp::server::TcpStreamServer>>>, tcp_server: Arc<OnceCell<Arc<transports::tcp::server::TcpStreamServer>>>,
system_status_server: Arc<OnceLock<Arc<system_status_server::SystemStatusServerInfo>>>, system_status_server: Arc<OnceLock<Arc<system_status_server::SystemStatusServerInfo>>>,
...@@ -104,15 +101,15 @@ impl DistributedRuntime { ...@@ -104,15 +101,15 @@ impl DistributedRuntime {
let runtime_clone = runtime.clone(); let runtime_clone = runtime.clone();
let store = match selected_kv_store { let store = match selected_kv_store {
KeyValueStoreSelect::Etcd(etcd_config) => { kv::Selector::Etcd(etcd_config) => {
let etcd_client = etcd::Client::new(*etcd_config, runtime_clone).await.inspect_err(|err| let etcd_client = etcd::Client::new(*etcd_config, runtime_clone).await.inspect_err(|err|
// The returned error doesn't show because of a dropped runtime error, so // The returned error doesn't show because of a dropped runtime error, so
// log it first. // log it first.
tracing::error!(%err, "Could not connect to etcd. Pass `--store-kv ..` to use a different backend or start etcd."))?; tracing::error!(%err, "Could not connect to etcd. Pass `--store-kv ..` to use a different backend or start etcd."))?;
KeyValueStoreManager::etcd(etcd_client) kv::Manager::etcd(etcd_client)
} }
KeyValueStoreSelect::File(root) => KeyValueStoreManager::file(root), kv::Selector::File(root) => kv::Manager::file(root),
KeyValueStoreSelect::Memory => KeyValueStoreManager::memory(), kv::Selector::Memory => kv::Manager::memory(),
}; };
let nats_client = match nats_config { let nats_client = match nats_config {
...@@ -377,7 +374,7 @@ impl DistributedRuntime { ...@@ -377,7 +374,7 @@ impl DistributedRuntime {
/// An interface to store things outside of the process. Usually backed by something like etcd. /// An interface to store things outside of the process. Usually backed by something like etcd.
/// Currently does key-value, but will grow to include whatever we need to store. /// Currently does key-value, but will grow to include whatever we need to store.
pub fn store(&self) -> &KeyValueStoreManager { pub fn store(&self) -> &kv::Manager {
&self.store &self.store
} }
...@@ -546,7 +543,7 @@ async fn nats_metrics_worker( ...@@ -546,7 +543,7 @@ async fn nats_metrics_worker(
#[derive(Dissolve)] #[derive(Dissolve)]
pub struct DistributedConfig { pub struct DistributedConfig {
pub store_backend: KeyValueStoreSelect, pub store_backend: kv::Selector,
pub nats_config: Option<nats::ClientOptions>, pub nats_config: Option<nats::ClientOptions>,
pub request_plane: RequestPlaneMode, pub request_plane: RequestPlaneMode,
} }
...@@ -555,7 +552,7 @@ impl DistributedConfig { ...@@ -555,7 +552,7 @@ impl DistributedConfig {
pub fn from_settings() -> DistributedConfig { pub fn from_settings() -> DistributedConfig {
let request_plane = RequestPlaneMode::from_env(); let request_plane = RequestPlaneMode::from_env();
DistributedConfig { DistributedConfig {
store_backend: KeyValueStoreSelect::Etcd(Box::default()), store_backend: kv::Selector::Etcd(Box::default()),
nats_config: if request_plane.is_nats() { nats_config: if request_plane.is_nats() {
Some(nats::ClientOptions::default()) Some(nats::ClientOptions::default())
} else { } else {
...@@ -572,7 +569,7 @@ impl DistributedConfig { ...@@ -572,7 +569,7 @@ impl DistributedConfig {
}; };
let request_plane = RequestPlaneMode::from_env(); let request_plane = RequestPlaneMode::from_env();
DistributedConfig { DistributedConfig {
store_backend: KeyValueStoreSelect::Etcd(Box::new(etcd_config)), store_backend: kv::Selector::Etcd(Box::new(etcd_config)),
nats_config: if request_plane.is_nats() { nats_config: if request_plane.is_nats() {
Some(nats::ClientOptions::default()) Some(nats::ClientOptions::default())
} else { } else {
...@@ -586,7 +583,7 @@ impl DistributedConfig { ...@@ -586,7 +583,7 @@ impl DistributedConfig {
/// same process. /// same process.
pub fn process_local() -> DistributedConfig { pub fn process_local() -> DistributedConfig {
DistributedConfig { DistributedConfig {
store_backend: KeyValueStoreSelect::Memory, store_backend: kv::Selector::Memory,
nats_config: None, nats_config: None,
// This won't be used in process local, so we likely need a "none" option to // This won't be used in process local, so we likely need a "none" option to
// communicate that and avoid opening the ports. // communicate that and avoid opening the ports.
...@@ -666,11 +663,11 @@ pub mod distributed_test_utils { ...@@ -666,11 +663,11 @@ pub mod distributed_test_utils {
/// Note: Settings are read from environment variables inside DistributedRuntime::from_settings /// Note: Settings are read from environment variables inside DistributedRuntime::from_settings
#[cfg(feature = "integration")] #[cfg(feature = "integration")]
pub async fn create_test_drt_async() -> super::DistributedRuntime { pub async fn create_test_drt_async() -> super::DistributedRuntime {
use crate::{storage::key_value_store::KeyValueStoreSelect, transports::nats}; use crate::{storage::kv, transports::nats};
let rt = crate::Runtime::from_current().unwrap(); let rt = crate::Runtime::from_current().unwrap();
let config = super::DistributedConfig { let config = super::DistributedConfig {
store_backend: KeyValueStoreSelect::Memory, store_backend: kv::Selector::Memory,
nats_config: Some(nats::ClientOptions::default()), nats_config: Some(nats::ClientOptions::default()),
request_plane: crate::distributed::RequestPlaneMode::default(), request_plane: crate::distributed::RequestPlaneMode::default(),
}; };
......
...@@ -53,10 +53,6 @@ pub use system_health::{HealthCheckTarget, SystemHealth}; ...@@ -53,10 +53,6 @@ pub use system_health::{HealthCheckTarget, SystemHealth};
pub use tokio_util::sync::CancellationToken; pub use tokio_util::sync::CancellationToken;
pub use worker::Worker; pub use worker::Worker;
use crate::{
metrics::prometheus_names::distributed_runtime, storage::key_value_store::KeyValueStore,
};
use component::Endpoint; use component::Endpoint;
use utils::GracefulShutdownTracker; use utils::GracefulShutdownTracker;
......
// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0 // SPDX-License-Identifier: Apache-2.0
pub mod key_value_store; pub mod kv;
...@@ -112,8 +112,8 @@ pub enum WatchEvent { ...@@ -112,8 +112,8 @@ pub enum WatchEvent {
} }
#[async_trait] #[async_trait]
pub trait KeyValueStore: Send + Sync { pub trait Store: Send + Sync {
type Bucket: KeyValueBucket + Send + Sync + 'static; type Bucket: Bucket + Send + Sync + 'static;
async fn get_or_create_bucket( async fn get_or_create_bucket(
&self, &self,
...@@ -130,7 +130,7 @@ pub trait KeyValueStore: Send + Sync { ...@@ -130,7 +130,7 @@ pub trait KeyValueStore: Send + Sync {
} }
#[derive(Clone, Debug, Default)] #[derive(Clone, Debug, Default)]
pub enum KeyValueStoreSelect { pub enum Selector {
// Box it because it is significantly bigger than the other variants // Box it because it is significantly bigger than the other variants
Etcd(Box<etcd_transport::ClientOptions>), Etcd(Box<etcd_transport::ClientOptions>),
File(PathBuf), File(PathBuf),
...@@ -139,23 +139,23 @@ pub enum KeyValueStoreSelect { ...@@ -139,23 +139,23 @@ pub enum KeyValueStoreSelect {
// Nats not listed because likely we want to remove that impl. It is not currently used and not well tested. // Nats not listed because likely we want to remove that impl. It is not currently used and not well tested.
} }
impl fmt::Display for KeyValueStoreSelect { impl fmt::Display for Selector {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self { match self {
KeyValueStoreSelect::Etcd(opts) => { Selector::Etcd(opts) => {
let urls = opts.etcd_url.join(","); let urls = opts.etcd_url.join(",");
write!(f, "Etcd({urls})") write!(f, "Etcd({urls})")
} }
KeyValueStoreSelect::File(path) => write!(f, "File({})", path.display()), Selector::File(path) => write!(f, "File({})", path.display()),
KeyValueStoreSelect::Memory => write!(f, "Memory"), Selector::Memory => write!(f, "Memory"),
} }
} }
} }
impl FromStr for KeyValueStoreSelect { impl FromStr for Selector {
type Err = anyhow::Error; type Err = anyhow::Error;
fn from_str(s: &str) -> anyhow::Result<KeyValueStoreSelect> { fn from_str(s: &str) -> anyhow::Result<Selector> {
match s { match s {
"etcd" => Ok(Self::Etcd(Box::default())), "etcd" => Ok(Self::Etcd(Box::default())),
"file" => { "file" => {
...@@ -170,16 +170,16 @@ impl FromStr for KeyValueStoreSelect { ...@@ -170,16 +170,16 @@ impl FromStr for KeyValueStoreSelect {
} }
} }
impl TryFrom<String> for KeyValueStoreSelect { impl TryFrom<String> for Selector {
type Error = anyhow::Error; type Error = anyhow::Error;
fn try_from(s: String) -> anyhow::Result<KeyValueStoreSelect> { fn try_from(s: String) -> anyhow::Result<Selector> {
s.parse() s.parse()
} }
} }
#[allow(clippy::large_enum_variant)] #[allow(clippy::large_enum_variant)]
pub enum KeyValueStoreEnum { enum KeyValueStoreEnum {
Memory(MemoryStore), Memory(MemoryStore),
Nats(NATSStore), Nats(NATSStore),
Etcd(EtcdStore), Etcd(EtcdStore),
...@@ -192,7 +192,7 @@ impl KeyValueStoreEnum { ...@@ -192,7 +192,7 @@ impl KeyValueStoreEnum {
bucket_name: &str, bucket_name: &str,
// auto-delete items older than this // auto-delete items older than this
ttl: Option<Duration>, ttl: Option<Duration>,
) -> Result<Box<dyn KeyValueBucket>, StoreError> { ) -> Result<Box<dyn Bucket>, StoreError> {
use KeyValueStoreEnum::*; use KeyValueStoreEnum::*;
Ok(match self { Ok(match self {
Memory(x) => Box::new(x.get_or_create_bucket(bucket_name, ttl).await?), Memory(x) => Box::new(x.get_or_create_bucket(bucket_name, ttl).await?),
...@@ -202,28 +202,25 @@ impl KeyValueStoreEnum { ...@@ -202,28 +202,25 @@ impl KeyValueStoreEnum {
}) })
} }
async fn get_bucket( async fn get_bucket(&self, bucket_name: &str) -> Result<Option<Box<dyn Bucket>>, StoreError> {
&self,
bucket_name: &str,
) -> Result<Option<Box<dyn KeyValueBucket>>, StoreError> {
use KeyValueStoreEnum::*; use KeyValueStoreEnum::*;
let maybe_bucket: Option<Box<dyn KeyValueBucket>> = match self { let maybe_bucket: Option<Box<dyn Bucket>> = match self {
Memory(x) => x Memory(x) => x
.get_bucket(bucket_name) .get_bucket(bucket_name)
.await? .await?
.map(|b| Box::new(b) as Box<dyn KeyValueBucket>), .map(|b| Box::new(b) as Box<dyn Bucket>),
Nats(x) => x Nats(x) => x
.get_bucket(bucket_name) .get_bucket(bucket_name)
.await? .await?
.map(|b| Box::new(b) as Box<dyn KeyValueBucket>), .map(|b| Box::new(b) as Box<dyn Bucket>),
Etcd(x) => x Etcd(x) => x
.get_bucket(bucket_name) .get_bucket(bucket_name)
.await? .await?
.map(|b| Box::new(b) as Box<dyn KeyValueBucket>), .map(|b| Box::new(b) as Box<dyn Bucket>),
File(x) => x File(x) => x
.get_bucket(bucket_name) .get_bucket(bucket_name)
.await? .await?
.map(|b| Box::new(b) as Box<dyn KeyValueBucket>), .map(|b| Box::new(b) as Box<dyn Bucket>),
}; };
Ok(maybe_bucket) Ok(maybe_bucket)
} }
...@@ -250,15 +247,15 @@ impl KeyValueStoreEnum { ...@@ -250,15 +247,15 @@ impl KeyValueStoreEnum {
} }
#[derive(Clone)] #[derive(Clone)]
pub struct KeyValueStoreManager(pub Arc<KeyValueStoreEnum>); pub struct Manager(Arc<KeyValueStoreEnum>);
impl Default for KeyValueStoreManager { impl Default for Manager {
fn default() -> Self { fn default() -> Self {
KeyValueStoreManager::memory() Manager::memory()
} }
} }
impl KeyValueStoreManager { impl Manager {
/// In-memory KeyValueStoreManager for testing /// In-memory KeyValueStoreManager for testing
pub fn memory() -> Self { pub fn memory() -> Self {
Self::new(KeyValueStoreEnum::Memory(MemoryStore::new())) Self::new(KeyValueStoreEnum::Memory(MemoryStore::new()))
...@@ -272,8 +269,8 @@ impl KeyValueStoreManager { ...@@ -272,8 +269,8 @@ impl KeyValueStoreManager {
Self::new(KeyValueStoreEnum::File(FileStore::new(root))) Self::new(KeyValueStoreEnum::File(FileStore::new(root)))
} }
fn new(s: KeyValueStoreEnum) -> KeyValueStoreManager { fn new(s: KeyValueStoreEnum) -> Manager {
KeyValueStoreManager(Arc::new(s)) Manager(Arc::new(s))
} }
pub async fn get_or_create_bucket( pub async fn get_or_create_bucket(
...@@ -281,14 +278,14 @@ impl KeyValueStoreManager { ...@@ -281,14 +278,14 @@ impl KeyValueStoreManager {
bucket_name: &str, bucket_name: &str,
// auto-delete items older than this // auto-delete items older than this
ttl: Option<Duration>, ttl: Option<Duration>,
) -> Result<Box<dyn KeyValueBucket>, StoreError> { ) -> Result<Box<dyn Bucket>, StoreError> {
self.0.get_or_create_bucket(bucket_name, ttl).await self.0.get_or_create_bucket(bucket_name, ttl).await
} }
pub async fn get_bucket( pub async fn get_bucket(
&self, &self,
bucket_name: &str, bucket_name: &str,
) -> Result<Option<Box<dyn KeyValueBucket>>, StoreError> { ) -> Result<Option<Box<dyn Bucket>>, StoreError> {
self.0.get_bucket(bucket_name).await self.0.get_bucket(bucket_name).await
} }
...@@ -397,7 +394,7 @@ impl KeyValueStoreManager { ...@@ -397,7 +394,7 @@ impl KeyValueStoreManager {
/// An online storage for key-value config values. /// An online storage for key-value config values.
#[async_trait] #[async_trait]
pub trait KeyValueBucket: Send + Sync { pub trait Bucket: Send + Sync {
/// A bucket is a collection of key/value pairs. /// A bucket is a collection of key/value pairs.
/// Insert a value into a bucket, if it doesn't exist already /// Insert a value into a bucket, if it doesn't exist already
/// The Key should be the name of the item, not including the bucket name. /// The Key should be the name of the item, not including the bucket name.
......
...@@ -5,15 +5,12 @@ use std::collections::HashMap; ...@@ -5,15 +5,12 @@ use std::collections::HashMap;
use std::pin::Pin; use std::pin::Pin;
use std::time::Duration; use std::time::Duration;
use crate::{ use crate::transports::etcd;
storage::key_value_store::{Key, KeyValue, WatchEvent},
transports::etcd,
};
use async_stream::stream; use async_stream::stream;
use async_trait::async_trait; use async_trait::async_trait;
use etcd_client::{Compare, CompareOp, EventType, PutOptions, Txn, TxnOp, WatchOptions}; use etcd_client::{Compare, CompareOp, EventType, PutOptions, Txn, TxnOp, WatchOptions};
use super::{KeyValueBucket, KeyValueStore, StoreError, StoreOutcome}; use super::{Bucket, Key, KeyValue, Store, StoreError, StoreOutcome, WatchEvent};
#[derive(Clone)] #[derive(Clone)]
pub struct EtcdStore { pub struct EtcdStore {
...@@ -27,7 +24,7 @@ impl EtcdStore { ...@@ -27,7 +24,7 @@ impl EtcdStore {
} }
#[async_trait] #[async_trait]
impl KeyValueStore for EtcdStore { impl Store for EtcdStore {
type Bucket = EtcdBucket; type Bucket = EtcdBucket;
/// A "bucket" in etcd is a path prefix /// A "bucket" in etcd is a path prefix
...@@ -66,7 +63,7 @@ pub struct EtcdBucket { ...@@ -66,7 +63,7 @@ pub struct EtcdBucket {
} }
#[async_trait] #[async_trait]
impl KeyValueBucket for EtcdBucket { impl Bucket for EtcdBucket {
async fn insert( async fn insert(
&self, &self,
key: &Key, key: &Key,
......
...@@ -21,9 +21,7 @@ use futures::StreamExt; ...@@ -21,9 +21,7 @@ use futures::StreamExt;
use notify::{Config, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher, event}; use notify::{Config, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher, event};
use parking_lot::Mutex; use parking_lot::Mutex;
use crate::storage::key_value_store::KeyValue; use super::{Bucket, Key, KeyValue, Store, StoreError, StoreOutcome, WatchEvent};
use super::{Key, KeyValueBucket, KeyValueStore, StoreError, StoreOutcome, WatchEvent};
/// How long until a key expires. We keep the keys alive by touching the files. /// How long until a key expires. We keep the keys alive by touching the files.
/// 10s is the same as our etcd lease expiry. /// 10s is the same as our etcd lease expiry.
...@@ -100,7 +98,7 @@ impl FileStore { ...@@ -100,7 +98,7 @@ impl FileStore {
} }
#[async_trait] #[async_trait]
impl KeyValueStore for FileStore { impl Store for FileStore {
type Bucket = Directory; type Bucket = Directory;
/// A "bucket" is a directory /// A "bucket" is a directory
...@@ -278,7 +276,7 @@ impl fmt::Display for Directory { ...@@ -278,7 +276,7 @@ impl fmt::Display for Directory {
} }
#[async_trait] #[async_trait]
impl KeyValueBucket for Directory { impl Bucket for Directory {
/// Write a file to the directory /// Write a file to the directory
async fn insert( async fn insert(
&self, &self,
...@@ -471,9 +469,7 @@ fn to_fs_err<E: std::error::Error>(err: E) -> StoreError { ...@@ -471,9 +469,7 @@ fn to_fs_err<E: std::error::Error>(err: E) -> StoreError {
mod tests { mod tests {
use std::collections::HashSet; use std::collections::HashSet;
use crate::storage::key_value_store::{ use crate::storage::kv::{Bucket as _, FileStore, Key, Store as _};
FileStore, Key, KeyValueBucket as _, KeyValueStore as _,
};
#[tokio::test] #[tokio::test]
async fn test_entries_full_path() { async fn test_entries_full_path() {
......
...@@ -11,9 +11,7 @@ use async_trait::async_trait; ...@@ -11,9 +11,7 @@ use async_trait::async_trait;
use rand::Rng as _; use rand::Rng as _;
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
use crate::storage::key_value_store::{Key, KeyValue, WatchEvent}; use super::{Bucket, Key, KeyValue, Store, StoreError, StoreOutcome, WatchEvent};
use super::{KeyValueBucket, KeyValueStore, StoreError, StoreOutcome};
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
enum MemoryEvent { enum MemoryEvent {
...@@ -71,7 +69,7 @@ impl MemoryStore { ...@@ -71,7 +69,7 @@ impl MemoryStore {
} }
#[async_trait] #[async_trait]
impl KeyValueStore for MemoryStore { impl Store for MemoryStore {
type Bucket = MemoryBucketRef; type Bucket = MemoryBucketRef;
async fn get_or_create_bucket( async fn get_or_create_bucket(
...@@ -112,7 +110,7 @@ impl KeyValueStore for MemoryStore { ...@@ -112,7 +110,7 @@ impl KeyValueStore for MemoryStore {
} }
#[async_trait] #[async_trait]
impl KeyValueBucket for MemoryBucketRef { impl Bucket for MemoryBucketRef {
async fn insert( async fn insert(
&self, &self,
key: &Key, key: &Key,
...@@ -233,12 +231,9 @@ impl KeyValueBucket for MemoryBucketRef { ...@@ -233,12 +231,9 @@ impl KeyValueBucket for MemoryBucketRef {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::storage::kv::{Bucket as _, Key, MemoryStore, Store as _};
use std::collections::HashSet; use std::collections::HashSet;
use crate::storage::key_value_store::{
Key, KeyValueBucket as _, KeyValueStore as _, MemoryStore,
};
#[tokio::test] #[tokio::test]
async fn test_entries_full_path() { async fn test_entries_full_path() {
let m = MemoryStore::new(); let m = MemoryStore::new();
......
...@@ -3,17 +3,12 @@ ...@@ -3,17 +3,12 @@
use std::{collections::HashMap, pin::Pin, time::Duration}; use std::{collections::HashMap, pin::Pin, time::Duration};
use crate::{ use crate::{protocols::EndpointId, slug::Slug, storage::kv, transports::nats::Client};
protocols::EndpointId,
slug::Slug,
storage::key_value_store::{Key, KeyValue, WatchEvent},
transports::nats::Client,
};
use async_nats::jetstream::kv::Operation; use async_nats::jetstream::kv::Operation;
use async_trait::async_trait; use async_trait::async_trait;
use futures::StreamExt; use futures::StreamExt;
use super::{KeyValueBucket, KeyValueStore, StoreError, StoreOutcome}; use super::{Bucket, Store, StoreError, StoreOutcome};
#[derive(Clone)] #[derive(Clone)]
pub struct NATSStore { pub struct NATSStore {
...@@ -26,7 +21,7 @@ pub struct NATSBucket { ...@@ -26,7 +21,7 @@ pub struct NATSBucket {
} }
#[async_trait] #[async_trait]
impl KeyValueStore for NATSStore { impl Store for NATSStore {
type Bucket = NATSBucket; type Bucket = NATSBucket;
async fn get_or_create_bucket( async fn get_or_create_bucket(
...@@ -120,10 +115,10 @@ impl NATSStore { ...@@ -120,10 +115,10 @@ impl NATSStore {
} }
#[async_trait] #[async_trait]
impl KeyValueBucket for NATSBucket { impl Bucket for NATSBucket {
async fn insert( async fn insert(
&self, &self,
key: &Key, key: &kv::Key,
value: bytes::Bytes, value: bytes::Bytes,
revision: u64, revision: u64,
) -> Result<StoreOutcome, StoreError> { ) -> Result<StoreOutcome, StoreError> {
...@@ -134,14 +129,14 @@ impl KeyValueBucket for NATSBucket { ...@@ -134,14 +129,14 @@ impl KeyValueBucket for NATSBucket {
} }
} }
async fn get(&self, key: &Key) -> Result<Option<bytes::Bytes>, StoreError> { async fn get(&self, key: &kv::Key) -> Result<Option<bytes::Bytes>, StoreError> {
self.nats_store self.nats_store
.get(key) .get(key)
.await .await
.map_err(|e| StoreError::NATSError(e.to_string())) .map_err(|e| StoreError::NATSError(e.to_string()))
} }
async fn delete(&self, key: &Key) -> Result<(), StoreError> { async fn delete(&self, key: &kv::Key) -> Result<(), StoreError> {
self.nats_store self.nats_store
.delete(key) .delete(key)
.await .await
...@@ -150,7 +145,8 @@ impl KeyValueBucket for NATSBucket { ...@@ -150,7 +145,8 @@ impl KeyValueBucket for NATSBucket {
async fn watch( async fn watch(
&self, &self,
) -> Result<Pin<Box<dyn futures::Stream<Item = WatchEvent> + Send + 'life0>>, StoreError> { ) -> Result<Pin<Box<dyn futures::Stream<Item = kv::WatchEvent> + Send + 'life0>>, StoreError>
{
let watch_stream = self let watch_stream = self
.nats_store .nats_store
.watch_all() .watch_all()
...@@ -165,15 +161,15 @@ impl KeyValueBucket for NATSBucket { ...@@ -165,15 +161,15 @@ impl KeyValueBucket for NATSBucket {
>| async move { >| async move {
match maybe_entry { match maybe_entry {
Ok(entry) => { Ok(entry) => {
let key = Key::new(entry.key); let key = kv::Key::new(entry.key);
Some(match entry.operation { Some(match entry.operation {
Operation::Put => { Operation::Put => {
let item = KeyValue::new(key, entry.value); let item = kv::KeyValue::new(key, entry.value);
WatchEvent::Put(item) kv::WatchEvent::Put(item)
} }
Operation::Delete => WatchEvent::Delete(key), Operation::Delete => kv::WatchEvent::Delete(key),
// TODO: What is Purge? Not urgent, NATS impl not used // TODO: What is Purge? Not urgent, NATS impl not used
Operation::Purge => WatchEvent::Delete(key), Operation::Purge => kv::WatchEvent::Delete(key),
}) })
} }
Err(e) => { Err(e) => {
...@@ -186,7 +182,7 @@ impl KeyValueBucket for NATSBucket { ...@@ -186,7 +182,7 @@ impl KeyValueBucket for NATSBucket {
)) ))
} }
async fn entries(&self) -> Result<HashMap<Key, bytes::Bytes>, StoreError> { async fn entries(&self) -> Result<HashMap<kv::Key, bytes::Bytes>, StoreError> {
let mut key_stream = self let mut key_stream = self
.nats_store .nats_store
.keys() .keys()
...@@ -195,7 +191,7 @@ impl KeyValueBucket for NATSBucket { ...@@ -195,7 +191,7 @@ impl KeyValueBucket for NATSBucket {
let mut out = HashMap::new(); let mut out = HashMap::new();
while let Some(Ok(key)) = key_stream.next().await { while let Some(Ok(key)) = key_stream.next().await {
if let Ok(Some(entry)) = self.nats_store.entry(&key).await { if let Ok(Some(entry)) = self.nats_store.entry(&key).await {
out.insert(Key::new(key), entry.value); out.insert(kv::Key::new(key), entry.value);
} }
} }
Ok(out) Ok(out)
...@@ -203,7 +199,7 @@ impl KeyValueBucket for NATSBucket { ...@@ -203,7 +199,7 @@ impl KeyValueBucket for NATSBucket {
} }
impl NATSBucket { impl NATSBucket {
async fn create(&self, key: &Key, value: bytes::Bytes) -> Result<StoreOutcome, StoreError> { async fn create(&self, key: &kv::Key, value: bytes::Bytes) -> Result<StoreOutcome, StoreError> {
match self.nats_store.create(&key, value).await { match self.nats_store.create(&key, value).await {
Ok(revision) => Ok(StoreOutcome::Created(revision)), Ok(revision) => Ok(StoreOutcome::Created(revision)),
Err(err) if err.kind() == async_nats::jetstream::kv::CreateErrorKind::AlreadyExists => { Err(err) if err.kind() == async_nats::jetstream::kv::CreateErrorKind::AlreadyExists => {
...@@ -226,7 +222,7 @@ impl NATSBucket { ...@@ -226,7 +222,7 @@ impl NATSBucket {
async fn update( async fn update(
&self, &self,
key: &Key, key: &kv::Key,
value: bytes::Bytes, value: bytes::Bytes,
revision: u64, revision: u64,
) -> Result<StoreOutcome, StoreError> { ) -> Result<StoreOutcome, StoreError> {
...@@ -246,7 +242,7 @@ impl NATSBucket { ...@@ -246,7 +242,7 @@ impl NATSBucket {
/// and try the update again. /// and try the update again.
async fn resync_update( async fn resync_update(
&self, &self,
key: &Key, key: &kv::Key,
value: bytes::Bytes, value: bytes::Bytes,
) -> Result<StoreOutcome, StoreError> { ) -> Result<StoreOutcome, StoreError> {
match self.nats_store.entry(key).await { match self.nats_store.entry(key).await {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment