feat: add python bindings + wheel build (#94)

Co-authored-by: Ryan McCormick <rmccormick@nvidia.com>
Co-authored-by: Neelay Shah <neelays@nvidia.com>

runtime/rust/python-wheel/examples/typed/client.py

+import asyncio
+
+from protocol import Request
+from triton_distributed_rs import DistributedRuntime, triton_worker
+
+
+@triton_worker()
+async def worker(runtime: DistributedRuntime):
+    """
+    Instantiate a `backend` client and call the `generate` endpoint
+    """
+    # get endpoint
+    endpoint = (
+        runtime.namespace("triton-init").component("backend").endpoint("generate")
+    )
+
+    # create client
+    client = await endpoint.client()
+
+    # list the endpoints
+    print(client.endpoint_ids())
+
+    # issue request
+    stream = await client.generate(Request(data="hello world").model_dump_json())
+
+    # process response
+    async for char in stream:
+        print(char)
+
+
+asyncio.run(worker())

runtime/rust/python-wheel/examples/typed/protocol.py

+from pydantic import BaseModel
+
+
+class Request(BaseModel):
+    data: str
+
+
+class Response(BaseModel):
+    char: str

runtime/rust/python-wheel/examples/typed/server.py

+import asyncio
+
+import uvloop
+from protocol import Request, Response
+from triton_distributed_rs import DistributedRuntime, triton_endpoint, triton_worker
+
+uvloop.install()
+
+
+class RequestHandler:
+    """
+    Request handler for the generate endpoint
+    """
+
+    @triton_endpoint(Request, Response)
+    async def generate(self, request):
+        for char in request.data:
+            yield char
+
+
+@triton_worker()
+async def worker(runtime: DistributedRuntime):
+    """
+    Instantiate a `backend` component and serve the `generate` endpoint
+    A `Component` can serve multiple endpoints
+    """
+    component = runtime.namespace("triton-init").component("backend")
+    await component.create_service()
+
+    endpoint = component.endpoint("generate")
+    await endpoint.serve_endpoint(RequestHandler().generate)
+
+
+asyncio.run(worker())

runtime/rust/python-wheel/pyproject.toml

+[project]
+name = "triton-distributed-rs"
+version = "0.1.1"
+description = "Distributed LLM Framework"
+# readme = "README.md"
+authors = [
+    { name = "Ryan Olson", email = "rolson@nvidia.com" }
+]
+requires-python = ">=3.10"
+dependencies = [
+    "pydantic>=2.10.6",
+    "uvloop>=0.21.0",
+]
+
+# [project.scripts]
+# triton-distributed = "triton_distributed_rs:main"
+
+[tool.maturin]
+module-name = "triton_distributed_rs._core"
+python-packages = ["triton_distributed_rs"]
+python-source = "python"
+
+[build-system]
+requires = ["maturin>=1.0,<2.0", "patchelf"]
+build-backend = "maturin"
+
+[tool.codespell]
+# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override -
+# this is only to allow you to run codespell interactively
+# this also overrides the grpc_generated folder, since it is generated
+
+# TODO add skip files for generated code
+# skip = "./.git,./.github,./src/grpc_generated"
+skip = "./.git,./.github"
+
+# ignore short words, and typename parameters like OffsetT
+ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b"
+# ignore allowed words
+# ignoring atleast to avoid testing::AtLeast from getting flagged
+ignore-words-list = "atleast"
+# use the 'clear' dictionary for unambiguous spelling mistakes
+builtin = "clear"
+# disable warnings about binary files and wrong encoding
+quiet-level = 3
+
+[tool.isort]
+profile = "black"
+use_parentheses = true
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+ensure_newline_before_comments = true
+line_length = 88
+balanced_wrapping = true
+indent = "    "
+
+[tool.ruff]
+# Same as Black.
+line-length = 88
+indent-width = 4
+
+
+[tool.mypy]
+
+# --disable-error-code: WAR large set of errors due to mypy not being run
+#   previously. We can slowly enable sets of errors to fix over time.
+# disable_error_code = []
+
+# --explicit-package-bases: WAR errors about duplicate module names used
+#   throughout project such as launch_workers.py
+# explicit_package_bases = true
+
+# --ignore-missing-imports: WAR too many errors when developing outside
+#   of container environment with PYTHONPATH set and packages installed.
+#   NOTE: Can possibly move mypy from pre-commit to a github action run only in
+#   a container with the expected environment and PYTHONPATH setup.
+ignore_missing_imports = true
+
+check_untyped_defs = true

runtime/rust/python-wheel/python/triton_distributed_rs/__init__.py

+import asyncio
+from functools import wraps
+from typing import Any, AsyncGenerator, Callable, Type
+
+from pydantic import BaseModel, ValidationError
+from triton_distributed_rs._core import DistributedRuntime
+
+
+def triton_worker():
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            loop = asyncio.get_running_loop()
+            runtime = DistributedRuntime(loop)
+
+            await func(runtime, *args, **kwargs)
+
+            # # wait for one of
+            # # 1. the task to complete
+            # # 2. the task to be cancelled
+
+            # done, pending = await asyncio.wait({task, cancelled}, return_when=asyncio.FIRST_COMPLETED)
+
+            # # i want to catch a SIGINT or SIGTERM or a cancellation event here
+
+            # try:
+            #     # Call the actual function
+            #     return await func(runtime, *args, **kwargs)
+            # finally:
+            #     print("Decorator: Cleaning up runtime resources")
+            #     # Perform cleanup actions here
+
+        return wrapper
+
+    return decorator
+
+
+def triton_endpoint(
+    request_model: Type[BaseModel], response_model: Type[BaseModel]
+) -> Callable:
+    def decorator(
+        func: Callable[..., AsyncGenerator[Any, None]]
+    ) -> Callable[..., AsyncGenerator[Any, None]]:
+        @wraps(func)
+        async def wrapper(*args, **kwargs) -> AsyncGenerator[Any, None]:
+            # Validate the request
+            try:
+                if len(args) in [1, 2]:
+                    args = list(args)
+                    args[-1] = request_model.parse_raw(args[-1])
+            except ValidationError as e:
+                raise ValueError(f"Invalid request: {e}")
+
+            # Wrap the async generator
+            async for item in func(*args, **kwargs):
+                # Validate the response
+                # TODO: Validate the response
+                try:
+                    yield item
+                except ValidationError as e:
+                    raise ValueError(f"Invalid response: {e}")
+
+        return wrapper
+
+    return decorator

runtime/rust/python-wheel/python/triton_distributed_rs/_core.pyi

+from typing import AsyncGenerator, AsyncIterator, Callable
+
+class JsonLike:
+    """
+    Any PyObject which can be serialized to JSON
+    """
+
+    ...
+
+RequestHandler = Callable[[JsonLike], AsyncGenerator[JsonLike, None]]
+
+class DistributedRuntime:
+    """
+    The runtime object for a distributed NOVA applications
+    """
+
+    ...
+
+    def namespace(self, name: str, path: str) -> Namespace:
+        """
+        Create a `Namespace` object
+        """
+        ...
+
+class Namespace:
+    """
+    A namespace is a collection of components
+    """
+
+    ...
+
+    def component(self, name: str) -> Component:
+        """
+        Create a `Component` object
+        """
+        ...
+
+class Component:
+    """
+    A component is a collection of endpoints
+    """
+
+    ...
+
+    def create_service(self) -> None:
+        """
+        Create a service
+        """
+        ...
+
+    def endpoint(self, name: str) -> Endpoint:
+        """
+        Create an endpoint
+        """
+        ...
+
+class Endpoint:
+    """
+    An Endpoint is a single API endpoint
+    """
+
+    ...
+
+    async def serve_endpoint(self, handler: RequestHandler) -> None:
+        """
+        Serve an endpoint discoverable by all connected clients at
+        `{{ namespace }}/components/{{ component_name }}/endpoints/{{ endpoint_name }}`
+        """
+        ...
+
+    async def client() -> Client:
+        """
+        Create a `Client` capable of calling served instances of this endpoint
+        """
+        ...
+
+class Client:
+    """
+    A client capable of calling served instances of an endpoint
+    """
+
+    ...
+
+    async def random(self, request: JsonLike) -> AsyncIterator[JsonLike]:
+        """
+        Pick a random instance of the endpoint and issue the request
+        """
+        ...
+
+    async def round_robin(self, request: JsonLike) -> AsyncIterator[JsonLike]:
+        """
+        Pick the next instance of the endpoint in a round-robin fashion
+        """
+        ...
+
+    async def direct(self, request: JsonLike, instance: str) -> AsyncIterator[JsonLike]:
+        """
+        Pick a specific instance of the endpoint
+        """
+        ...

runtime/rust/python-wheel/rust/engine.rs

+pub use serde::{Deserialize, Serialize};
+pub use triton_distributed::{
+    error,
+    pipeline::{
+        async_trait, AsyncEngine, AsyncEngineContextProvider, Data, ManyOut, ResponseStream,
+        SingleIn,
+    },
+    protocols::annotated::Annotated,
+    Error, Result,
+};
+
+use pyo3::prelude::*;
+use pyo3_async_runtimes::TaskLocals;
+use pythonize::{depythonize, pythonize};
+
+use tokio::sync::mpsc;
+use tokio_stream::{wrappers::ReceiverStream, StreamExt};
+use tracing as log;
+
+/// Add bingings from this crate to the provided module
+pub fn add_to_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_class::<PythonAsyncEngine>()?;
+    Ok(())
+}
+
+// todos:
+// - [ ] enable context cancellation
+//   - this will likely require a change to the function signature python calling arguments
+// - [ ] rename `PythonAsyncEngine` to `PythonServerStreamingEngine` to be more descriptive
+// - [ ] other `AsyncEngine` implementations will have a similar pattern, i.e. one AsyncEngine
+//       implementation per struct
+
+/// Rust/Python bridge that maps to the [`AsyncEngine`] trait
+///
+/// Currently this is only implemented for the [`SingleIn`] and [`ManyOut`] types; however,
+/// more [`AsyncEngine`] implementations can be added in the future.
+///
+/// For the [`SingleIn`] and [`ManyOut`] case, this implementation will take a Python async
+/// generator and convert it to a Rust async stream.
+///
+/// ```python
+/// class ComputeEngine:
+///     def __init__(self):
+///         self.compute_engine = make_compute_engine()
+///
+///     def generate(self, request):
+///         async generator():
+///            async for output in self.compute_engine.generate(request):
+///                yield output
+///         return generator()
+///
+/// def main():
+///     loop = asyncio.create_event_loop()
+///     compute_engine = ComputeEngine()
+///     engine = PythonAsyncEngine(compute_engine.generate, loop)
+///     service = RustService()
+///     service.add_engine("model_name", engine)
+///     loop.run_until_complete(service.run())
+/// ```
+#[pyclass]
+#[derive(Clone)]
+pub struct PythonAsyncEngine {
+    generator: PyObject,
+    event_loop: PyObject,
+}
+
+#[pymethods]
+impl PythonAsyncEngine {
+    /// Create a new instance of the PythonAsyncEngine
+    ///
+    /// # Arguments
+    /// - `generator`: a Python async generator that will be used to generate responses
+    /// - `event_loop`: the Python event loop that will be used to run the generator
+    ///
+    /// Note: In Rust land, the request and the response are both concrete; however, in
+    /// Python land, the request and response not strongly typed, meaning the generator
+    /// could accept a different type of request or return a different type of response
+    /// and we would not know until runtime.
+    #[new]
+    pub fn new(generator: PyObject, event_loop: PyObject) -> PyResult<Self> {
+        Ok(PythonAsyncEngine {
+            generator,
+            event_loop,
+        })
+    }
+}
+
+#[async_trait]
+impl<Req, Resp> AsyncEngine<SingleIn<Req>, ManyOut<Annotated<Resp>>, Error> for PythonAsyncEngine
+where
+    Req: Data + Serialize,
+    Resp: Data + for<'de> Deserialize<'de>,
+{
+    async fn generate(&self, request: SingleIn<Req>) -> Result<ManyOut<Annotated<Resp>>, Error> {
+        // Create a context
+        let (request, context) = request.transfer(());
+
+        let id = context.id().to_string();
+        log::trace!("processing request: {}", id);
+
+        // Clone the PyObject to move into the thread
+
+        // Create a channel to communicate between the Python thread and the Rust async context
+        let (tx, rx) = mpsc::channel::<Annotated<Resp>>(128);
+        let tx_error = tx.clone();
+
+        let stream = Python::with_gil(|py| {
+            let py_request = pythonize(py, &request)?;
+            let gen = self.generator.call1(py, (py_request,))?;
+            let locals = TaskLocals::new(self.event_loop.bind(py).clone());
+            pyo3_async_runtimes::tokio::into_stream_with_locals_v1(locals, gen.into_bound(py))
+        })?;
+
+        let stream = Box::pin(stream);
+
+        let process = |item: Result<Py<PyAny>, PyErr>| -> Result<Annotated<Resp>, Error> {
+            let item = item
+                .map_err(|err| error!("error processing python async generator stream: {}", err))?;
+
+            let response = Python::with_gil(|py| depythonize::<Resp>(&item.into_bound(py)))?;
+            let response = Annotated::from_data(response);
+
+            Ok(response)
+        };
+
+        // process the stream
+        // any error thrown in the stream will be caught and complete the processing task
+        // errors are captured by a task that is watching the processing task
+        // the error will be emitted as an annotated error
+        let processor = tokio::spawn(async move {
+            log::trace!("processing stream from python async generator: {}", id);
+            let mut stream = stream;
+
+            while let Some(item) = stream.next().await {
+                // let mut done = false;
+                let response = match process(item) {
+                    Ok(response) => response,
+                    Err(err) => {
+                        // done = true;
+                        Annotated::from_error(err.to_string())
+                    }
+                };
+
+                if tx.send(response).await.is_err() {
+                    log::error!("generator response channel was dropped: {}", id);
+                    return Err(error!("generator response channel was dropped"));
+                }
+
+                // if done {
+                //     break;
+                // }
+            }
+
+            Result::<()>::Ok(())
+        });
+
+        tokio::spawn(async move {
+            match processor.await {
+                Ok(Ok(_)) => {}
+                Ok(Err(err)) => {
+                    log::error!("error processing python async generator: {}", err);
+                    tx_error
+                        .send(Annotated::from_error(err.to_string()))
+                        .await
+                        .unwrap();
+                }
+                Err(err) => {
+                    log::error!(
+                        "error on tokio task for processing python async generator stream: {}",
+                        err
+                    );
+                    tx_error
+                        .send(Annotated::from_error(err.to_string()))
+                        .await
+                        .unwrap();
+                }
+            }
+        });
+
+        let stream = ReceiverStream::new(rx);
+
+        Ok(ResponseStream::new(Box::pin(stream), context.context()))
+    }
+}

runtime/rust/python-wheel/rust/lib.rs

+use futures::StreamExt;
+use once_cell::sync::OnceCell;
+use pyo3::exceptions::PyStopAsyncIteration;
+use pyo3::types::PyString;
+use pyo3::IntoPyObjectExt;
+use pyo3::{exceptions::PyException, prelude::*};
+use rs::pipeline::network::Ingress;
+use std::{fmt::Display, sync::Arc};
+use tokio::sync::Mutex;
+use tracing as log;
+
+use triton_distributed::{
+    self as rs,
+    pipeline::{EngineStream, ManyOut, SingleIn},
+    protocols::annotated::Annotated as RsAnnotated,
+};
+
+mod engine;
+
+type JsonServerStreamingIngress =
+    Ingress<SingleIn<serde_json::Value>, ManyOut<RsAnnotated<serde_json::Value>>>;
+
+static INIT: OnceCell<()> = OnceCell::new();
+
+const DEFAULT_ANNOTATED_SETTING: Option<bool> = Some(true);
+
+/// A Python module implemented in Rust. The name of this function must match
+/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to
+/// import the module.
+#[pymodule]
+fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_class::<DistributedRuntime>()?;
+    m.add_class::<CancellationToken>()?;
+    m.add_class::<Namespace>()?;
+    m.add_class::<Component>()?;
+    m.add_class::<Endpoint>()?;
+    m.add_class::<Client>()?;
+    m.add_class::<AsyncResponseStream>()?;
+
+    engine::add_to_module(m)?;
+
+    // llm::http::add_to_module(m)?;
+
+    Ok(())
+}
+
+pub fn to_pyerr<E>(err: E) -> PyErr
+where
+    E: Display,
+{
+    PyException::new_err(format!("{}", err))
+}
+
+#[pyclass]
+#[derive(Clone)]
+struct DistributedRuntime {
+    inner: rs::DistributedRuntime,
+    event_loop: PyObject,
+}
+
+#[derive(Clone)]
+#[pyclass]
+struct CancellationToken {
+    inner: rs::CancellationToken,
+}
+
+#[pyclass]
+struct Namespace {
+    inner: rs::component::Namespace,
+    event_loop: PyObject,
+}
+
+#[pyclass]
+struct Component {
+    inner: rs::component::Component,
+    event_loop: PyObject,
+}
+
+#[pyclass]
+struct Endpoint {
+    inner: rs::component::Endpoint,
+    event_loop: PyObject,
+}
+
+#[pyclass]
+struct Client {
+    inner: rs::component::Client<serde_json::Value, serde_json::Value>,
+}
+
+#[pymethods]
+impl DistributedRuntime {
+    #[new]
+    fn new(event_loop: PyObject) -> PyResult<Self> {
+        let rt = rs::Worker::from_settings().map_err(to_pyerr)?;
+        INIT.get_or_try_init(|| {
+            let primary = rt.tokio_runtime()?;
+            pyo3_async_runtimes::tokio::init_with_runtime(primary)
+                .map_err(|e| rs::error!("failed to initialize pyo3 static runtime: {:?}", e))?;
+            rs::OK(())
+        })
+        .map_err(to_pyerr)?;
+
+        let runtime = rt.runtime().clone();
+
+        let inner = rt
+            .runtime()
+            .secondary()
+            .block_on(rs::DistributedRuntime::from_settings(runtime))
+            .map_err(to_pyerr)?;
+
+        Ok(DistributedRuntime { inner, event_loop })
+    }
+
+    fn namespace(&self, name: String) -> PyResult<Namespace> {
+        Ok(Namespace {
+            inner: self.inner.namespace(name).map_err(to_pyerr)?,
+            event_loop: self.event_loop.clone(),
+        })
+    }
+
+    fn primary_token(&self) -> CancellationToken {
+        let inner = self.inner.runtime().primary_token();
+        CancellationToken { inner }
+    }
+
+    fn child_token(&self) -> CancellationToken {
+        let inner = self.inner.runtime().child_token();
+        CancellationToken { inner }
+    }
+
+    fn shutdown(&self) {
+        self.inner.runtime().shutdown();
+    }
+
+    fn event_loop(&self) -> PyObject {
+        self.event_loop.clone()
+    }
+}
+
+#[pymethods]
+impl CancellationToken {
+    fn cancel(&self) {
+        self.inner.cancel();
+    }
+
+    fn cancelled<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
+        let token = self.inner.clone();
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            token.cancelled().await;
+            Ok(())
+        })
+    }
+}
+
+#[pymethods]
+impl Component {
+    fn endpoint(&self, name: String) -> PyResult<Endpoint> {
+        let inner = self.inner.endpoint(name);
+        Ok(Endpoint {
+            inner,
+            event_loop: self.event_loop.clone(),
+        })
+    }
+
+    fn create_service<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
+        let builder = self.inner.service_builder();
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let _ = builder.create().await.map_err(to_pyerr)?;
+            Ok(())
+        })
+    }
+}
+
+#[pymethods]
+impl Endpoint {
+    fn serve_endpoint<'p>(
+        &self,
+        py: Python<'p>,
+        generator: PyObject,
+    ) -> PyResult<Bound<'p, PyAny>> {
+        let engine = Arc::new(engine::PythonAsyncEngine::new(
+            generator,
+            self.event_loop.clone(),
+        )?);
+        let ingress = JsonServerStreamingIngress::for_engine(engine).map_err(to_pyerr)?;
+        let builder = self.inner.endpoint_builder().handler(ingress);
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            builder.start().await.map_err(to_pyerr)?;
+            Ok(())
+        })
+    }
+
+    fn client<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
+        let inner = self.inner.clone();
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let client = inner
+                .client::<serde_json::Value, serde_json::Value>()
+                .await
+                .map_err(to_pyerr)?;
+            Ok(Client { inner: client })
+        })
+    }
+}
+
+#[pymethods]
+impl Namespace {
+    fn component(&self, name: String) -> PyResult<Component> {
+        let inner = self.inner.component(name).map_err(to_pyerr)?;
+        Ok(Component {
+            inner,
+            event_loop: self.event_loop.clone(),
+        })
+    }
+}
+
+#[pymethods]
+impl Client {
+    /// Get list of current endpoints
+    fn endpoint_ids(&self) -> Vec<i64> {
+        self.inner.endpoint_ids().borrow().clone()
+    }
+
+    fn wait_for_endpoints<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
+        let inner = self.inner.clone();
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            inner.wait_for_endpoints().await.map_err(to_pyerr)
+        })
+    }
+
+    /// Issue a request to the endpoint using the default routing strategy.
+    #[pyo3(signature = (request, annotated=DEFAULT_ANNOTATED_SETTING))]
+    fn generate<'p>(
+        &self,
+        py: Python<'p>,
+        request: PyObject,
+        annotated: Option<bool>,
+    ) -> PyResult<Bound<'p, PyAny>> {
+        self.random(py, request, annotated)
+    }
+
+    /// Send a request to the next endpoint in a round-robin fashion.
+    #[pyo3(signature = (request, annotated=DEFAULT_ANNOTATED_SETTING))]
+    fn round_robin<'p>(
+        &self,
+        py: Python<'p>,
+        request: PyObject,
+        annotated: Option<bool>,
+    ) -> PyResult<Bound<'p, PyAny>> {
+        let request: serde_json::Value = pythonize::depythonize(&request.into_bound(py))?;
+        let annotated = annotated.unwrap_or(false);
+
+        let (tx, rx) = tokio::sync::mpsc::channel(32);
+        let client = self.inner.clone();
+
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let stream = client.round_robin(request.into()).await.map_err(to_pyerr)?;
+            tokio::spawn(process_stream(stream, tx));
+            Ok(AsyncResponseStream {
+                rx: Arc::new(Mutex::new(rx)),
+                annotated,
+            })
+        })
+    }
+
+    /// Send a request to a random endpoint.
+    #[pyo3(signature = (request, annotated=DEFAULT_ANNOTATED_SETTING))]
+    fn random<'p>(
+        &self,
+        py: Python<'p>,
+        request: PyObject,
+        annotated: Option<bool>,
+    ) -> PyResult<Bound<'p, PyAny>> {
+        let request: serde_json::Value = pythonize::depythonize(&request.into_bound(py))?;
+        let annotated = annotated.unwrap_or(false);
+
+        let (tx, rx) = tokio::sync::mpsc::channel(32);
+        let client = self.inner.clone();
+
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let stream = client.random(request.into()).await.map_err(to_pyerr)?;
+            tokio::spawn(process_stream(stream, tx));
+            Ok(AsyncResponseStream {
+                rx: Arc::new(Mutex::new(rx)),
+                annotated,
+            })
+        })
+    }
+
+    /// Directly send a request to a specific endpoint.
+    #[pyo3(signature = (request, endpoint_id, annotated=DEFAULT_ANNOTATED_SETTING))]
+    fn direct<'p>(
+        &self,
+        py: Python<'p>,
+        request: PyObject,
+        endpoint_id: i64,
+        annotated: Option<bool>,
+    ) -> PyResult<Bound<'p, PyAny>> {
+        let request: serde_json::Value = pythonize::depythonize(&request.into_bound(py))?;
+        let annotated = annotated.unwrap_or(false);
+
+        let (tx, rx) = tokio::sync::mpsc::channel(32);
+        let client = self.inner.clone();
+
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let stream = client
+                .direct(request.into(), endpoint_id)
+                .await
+                .map_err(to_pyerr)?;
+
+            tokio::spawn(process_stream(stream, tx));
+
+            Ok(AsyncResponseStream {
+                rx: Arc::new(Mutex::new(rx)),
+                annotated,
+            })
+        })
+    }
+}
+
+async fn process_stream(
+    stream: EngineStream<serde_json::Value>,
+    tx: tokio::sync::mpsc::Sender<RsAnnotated<PyObject>>,
+) {
+    let mut stream = stream;
+    while let Some(response) = stream.next().await {
+        // Convert the response to a PyObject using Python's GIL
+        let annotated: RsAnnotated<serde_json::Value> = serde_json::from_value(response).unwrap();
+
+        let annotated: RsAnnotated<PyObject> = annotated.map_data(|data| {
+            let result = Python::with_gil(|py| match pythonize::pythonize(py, &data) {
+                Ok(pyobj) => Ok(pyobj.into()),
+                Err(e) => Err(e.to_string()),
+            });
+            result
+        });
+
+        let is_error = annotated.is_error();
+
+        // Send the PyObject through the channel or log an error
+        if let Err(e) = tx.send(annotated).await {
+            log::error!("Failed to send response: {:?}", e);
+        }
+
+        if is_error {
+            break;
+        }
+    }
+}
+
+#[pyclass]
+struct AsyncResponseStream {
+    rx: Arc<Mutex<tokio::sync::mpsc::Receiver<RsAnnotated<PyObject>>>>,
+    annotated: bool,
+}
+
+#[pymethods]
+impl AsyncResponseStream {
+    /// This method is required to implement the `AsyncIterator` protocol.
+    #[pyo3(name = "__aiter__")]
+    fn aiter(slf: PyRef<Self>, py: Python) -> PyResult<Py<PyAny>> {
+        slf.into_py_any(py)
+    }
+    /// This method is required to implement the `AsyncIterator` protocol.
+    #[pyo3(name = "__anext__")]
+    fn next<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
+        let rx = self.rx.clone();
+        let annotated = self.annotated;
+
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            loop {
+                let value = rx.lock().await.recv().await;
+                match value {
+                    Some(pyobj) => {
+                        let pyobj = match pyobj.ok() {
+                            Ok(pyobj) => pyobj,
+                            Err(e) => {
+                                return Err(PyErr::new::<pyo3::exceptions::PyValueError, _>(e));
+                            }
+                        };
+
+                        if annotated {
+                            let object = Annotated { inner: pyobj };
+                            #[allow(deprecated)]
+                            let object = Python::with_gil(|py| object.into_py(py));
+                            return Ok(object);
+                        } else {
+                            match pyobj.data {
+                                Some(data) => return Ok(data),
+                                None => continue,
+                            }
+                        }
+                    }
+                    None => return Err(PyStopAsyncIteration::new_err("Stream exhausted")),
+                }
+            }
+        })
+    }
+}
+
+#[pyclass]
+struct Annotated {
+    inner: RsAnnotated<PyObject>,
+}
+
+#[pymethods]
+impl Annotated {
+    #[new]
+    fn new(data: PyObject) -> Self {
+        Annotated {
+            inner: RsAnnotated::from_data(data),
+        }
+    }
+
+    fn is_error(&self) -> bool {
+        self.inner.is_error()
+    }
+
+    fn data(&self) -> Option<PyObject> {
+        self.inner.data.clone()
+    }
+
+    fn event(&self) -> Option<String> {
+        self.inner.event.clone()
+    }
+
+    fn comments(&self) -> Option<Vec<String>> {
+        self.inner.comment.clone()
+    }
+
+    fn id(&self) -> Option<String> {
+        self.inner.id.clone()
+    }
+
+    #[pyo3(name = "__repr__")]
+    fn _repr(&self, py: Python) -> String {
+        let data = self.inner.data.clone().map(|obj| {
+            obj.call_method0(py, "__repr__")
+                .and_then(|repr_obj| repr_obj.extract::<Py<PyString>>(py))
+                .map(|py_str| py_str.to_string_lossy(py).into_owned())
+                .unwrap_or_else(|_| "<failed_repr>".to_string())
+        });
+
+        format!(
+            "Annotated(data={}, event={}, comment={:?}, id={})",
+            data.unwrap_or_else(|| "<no_data>".to_string()),
+            self.inner.event.as_deref().unwrap_or("None"),
+            self.inner.comment.as_deref().unwrap_or(&[]),
+            self.inner.id.as_deref().unwrap_or("None")
+        )
+    }
+}

runtime/rust/python-wheel/uv.lock

+version = 1
+requires-python = ">=3.10"
+
+[[package]]
+name = "annotated-types"
+version = "0.7.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 },
+]
+
+[[package]]
+name = "triton-distributed"
+version = "0.1.0"
+source = { editable = "." }
+dependencies = [
+    { name = "pydantic" },
+    { name = "uvloop" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "pydantic", specifier = ">=2.10.6" },
+    { name = "uvloop", specifier = ">=0.21.0" },
+]
+
+[[package]]
+name = "pydantic"
+version = "2.10.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "annotated-types" },
+    { name = "pydantic-core" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b7/ae/d5220c5c52b158b1de7ca89fc5edb72f304a70a4c540c84c8844bf4008de/pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236", size = 761681 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f4/3c/8cc1cc84deffa6e25d2d0c688ebb80635dfdbf1dbea3e30c541c8cf4d860/pydantic-2.10.6-py3-none-any.whl", hash = "sha256:427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584", size = 431696 },
+]
+
+[[package]]
+name = "pydantic-core"
+version = "2.27.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fc/01/f3e5ac5e7c25833db5eb555f7b7ab24cd6f8c322d3a3ad2d67a952dc0abc/pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39", size = 413443 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3a/bc/fed5f74b5d802cf9a03e83f60f18864e90e3aed7223adaca5ffb7a8d8d64/pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa", size = 1895938 },
+    { url = "https://files.pythonhosted.org/packages/71/2a/185aff24ce844e39abb8dd680f4e959f0006944f4a8a0ea372d9f9ae2e53/pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c", size = 1815684 },
+    { url = "https://files.pythonhosted.org/packages/c3/43/fafabd3d94d159d4f1ed62e383e264f146a17dd4d48453319fd782e7979e/pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a", size = 1829169 },
+    { url = "https://files.pythonhosted.org/packages/a2/d1/f2dfe1a2a637ce6800b799aa086d079998959f6f1215eb4497966efd2274/pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5", size = 1867227 },
+    { url = "https://files.pythonhosted.org/packages/7d/39/e06fcbcc1c785daa3160ccf6c1c38fea31f5754b756e34b65f74e99780b5/pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c", size = 2037695 },
+    { url = "https://files.pythonhosted.org/packages/7a/67/61291ee98e07f0650eb756d44998214231f50751ba7e13f4f325d95249ab/pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7", size = 2741662 },
+    { url = "https://files.pythonhosted.org/packages/32/90/3b15e31b88ca39e9e626630b4c4a1f5a0dfd09076366f4219429e6786076/pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a", size = 1993370 },
+    { url = "https://files.pythonhosted.org/packages/ff/83/c06d333ee3a67e2e13e07794995c1535565132940715931c1c43bfc85b11/pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236", size = 1996813 },
+    { url = "https://files.pythonhosted.org/packages/7c/f7/89be1c8deb6e22618a74f0ca0d933fdcb8baa254753b26b25ad3acff8f74/pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962", size = 2005287 },
+    { url = "https://files.pythonhosted.org/packages/b7/7d/8eb3e23206c00ef7feee17b83a4ffa0a623eb1a9d382e56e4aa46fd15ff2/pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9", size = 2128414 },
+    { url = "https://files.pythonhosted.org/packages/4e/99/fe80f3ff8dd71a3ea15763878d464476e6cb0a2db95ff1c5c554133b6b83/pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af", size = 2155301 },
+    { url = "https://files.pythonhosted.org/packages/2b/a3/e50460b9a5789ca1451b70d4f52546fa9e2b420ba3bfa6100105c0559238/pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4", size = 1816685 },
+    { url = "https://files.pythonhosted.org/packages/57/4c/a8838731cb0f2c2a39d3535376466de6049034d7b239c0202a64aaa05533/pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31", size = 1982876 },
+    { url = "https://files.pythonhosted.org/packages/c2/89/f3450af9d09d44eea1f2c369f49e8f181d742f28220f88cc4dfaae91ea6e/pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc", size = 1893421 },
+    { url = "https://files.pythonhosted.org/packages/9e/e3/71fe85af2021f3f386da42d291412e5baf6ce7716bd7101ea49c810eda90/pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7", size = 1814998 },
+    { url = "https://files.pythonhosted.org/packages/a6/3c/724039e0d848fd69dbf5806894e26479577316c6f0f112bacaf67aa889ac/pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15", size = 1826167 },
+    { url = "https://files.pythonhosted.org/packages/2b/5b/1b29e8c1fb5f3199a9a57c1452004ff39f494bbe9bdbe9a81e18172e40d3/pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306", size = 1865071 },
+    { url = "https://files.pythonhosted.org/packages/89/6c/3985203863d76bb7d7266e36970d7e3b6385148c18a68cc8915fd8c84d57/pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99", size = 2036244 },
+    { url = "https://files.pythonhosted.org/packages/0e/41/f15316858a246b5d723f7d7f599f79e37493b2e84bfc789e58d88c209f8a/pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459", size = 2737470 },
+    { url = "https://files.pythonhosted.org/packages/a8/7c/b860618c25678bbd6d1d99dbdfdf0510ccb50790099b963ff78a124b754f/pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048", size = 1992291 },
+    { url = "https://files.pythonhosted.org/packages/bf/73/42c3742a391eccbeab39f15213ecda3104ae8682ba3c0c28069fbcb8c10d/pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d", size = 1994613 },
+    { url = "https://files.pythonhosted.org/packages/94/7a/941e89096d1175d56f59340f3a8ebaf20762fef222c298ea96d36a6328c5/pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b", size = 2002355 },
+    { url = "https://files.pythonhosted.org/packages/6e/95/2359937a73d49e336a5a19848713555605d4d8d6940c3ec6c6c0ca4dcf25/pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474", size = 2126661 },
+    { url = "https://files.pythonhosted.org/packages/2b/4c/ca02b7bdb6012a1adef21a50625b14f43ed4d11f1fc237f9d7490aa5078c/pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6", size = 2153261 },
+    { url = "https://files.pythonhosted.org/packages/72/9d/a241db83f973049a1092a079272ffe2e3e82e98561ef6214ab53fe53b1c7/pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c", size = 1812361 },
+    { url = "https://files.pythonhosted.org/packages/e8/ef/013f07248041b74abd48a385e2110aa3a9bbfef0fbd97d4e6d07d2f5b89a/pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc", size = 1982484 },
+    { url = "https://files.pythonhosted.org/packages/10/1c/16b3a3e3398fd29dca77cea0a1d998d6bde3902fa2706985191e2313cc76/pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4", size = 1867102 },
+    { url = "https://files.pythonhosted.org/packages/d6/74/51c8a5482ca447871c93e142d9d4a92ead74de6c8dc5e66733e22c9bba89/pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0", size = 1893127 },
+    { url = "https://files.pythonhosted.org/packages/d3/f3/c97e80721735868313c58b89d2de85fa80fe8dfeeed84dc51598b92a135e/pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef", size = 1811340 },
+    { url = "https://files.pythonhosted.org/packages/9e/91/840ec1375e686dbae1bd80a9e46c26a1e0083e1186abc610efa3d9a36180/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7", size = 1822900 },
+    { url = "https://files.pythonhosted.org/packages/f6/31/4240bc96025035500c18adc149aa6ffdf1a0062a4b525c932065ceb4d868/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934", size = 1869177 },
+    { url = "https://files.pythonhosted.org/packages/fa/20/02fbaadb7808be578317015c462655c317a77a7c8f0ef274bc016a784c54/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6", size = 2038046 },
+    { url = "https://files.pythonhosted.org/packages/06/86/7f306b904e6c9eccf0668248b3f272090e49c275bc488a7b88b0823444a4/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c", size = 2685386 },
+    { url = "https://files.pythonhosted.org/packages/8d/f0/49129b27c43396581a635d8710dae54a791b17dfc50c70164866bbf865e3/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2", size = 1997060 },
+    { url = "https://files.pythonhosted.org/packages/0d/0f/943b4af7cd416c477fd40b187036c4f89b416a33d3cc0ab7b82708a667aa/pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4", size = 2004870 },
+    { url = "https://files.pythonhosted.org/packages/35/40/aea70b5b1a63911c53a4c8117c0a828d6790483f858041f47bab0b779f44/pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3", size = 1999822 },
+    { url = "https://files.pythonhosted.org/packages/f2/b3/807b94fd337d58effc5498fd1a7a4d9d59af4133e83e32ae39a96fddec9d/pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4", size = 2130364 },
+    { url = "https://files.pythonhosted.org/packages/fc/df/791c827cd4ee6efd59248dca9369fb35e80a9484462c33c6649a8d02b565/pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57", size = 2158303 },
+    { url = "https://files.pythonhosted.org/packages/9b/67/4e197c300976af185b7cef4c02203e175fb127e414125916bf1128b639a9/pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc", size = 1834064 },
+    { url = "https://files.pythonhosted.org/packages/1f/ea/cd7209a889163b8dcca139fe32b9687dd05249161a3edda62860430457a5/pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9", size = 1989046 },
+    { url = "https://files.pythonhosted.org/packages/bc/49/c54baab2f4658c26ac633d798dab66b4c3a9bbf47cff5284e9c182f4137a/pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b", size = 1885092 },
+    { url = "https://files.pythonhosted.org/packages/41/b1/9bc383f48f8002f99104e3acff6cba1231b29ef76cfa45d1506a5cad1f84/pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b", size = 1892709 },
+    { url = "https://files.pythonhosted.org/packages/10/6c/e62b8657b834f3eb2961b49ec8e301eb99946245e70bf42c8817350cbefc/pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154", size = 1811273 },
+    { url = "https://files.pythonhosted.org/packages/ba/15/52cfe49c8c986e081b863b102d6b859d9defc63446b642ccbbb3742bf371/pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9", size = 1823027 },
+    { url = "https://files.pythonhosted.org/packages/b1/1c/b6f402cfc18ec0024120602bdbcebc7bdd5b856528c013bd4d13865ca473/pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9", size = 1868888 },
+    { url = "https://files.pythonhosted.org/packages/bd/7b/8cb75b66ac37bc2975a3b7de99f3c6f355fcc4d89820b61dffa8f1e81677/pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1", size = 2037738 },
+    { url = "https://files.pythonhosted.org/packages/c8/f1/786d8fe78970a06f61df22cba58e365ce304bf9b9f46cc71c8c424e0c334/pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a", size = 2685138 },
+    { url = "https://files.pythonhosted.org/packages/a6/74/d12b2cd841d8724dc8ffb13fc5cef86566a53ed358103150209ecd5d1999/pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e", size = 1997025 },
+    { url = "https://files.pythonhosted.org/packages/a0/6e/940bcd631bc4d9a06c9539b51f070b66e8f370ed0933f392db6ff350d873/pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4", size = 2004633 },
+    { url = "https://files.pythonhosted.org/packages/50/cc/a46b34f1708d82498c227d5d80ce615b2dd502ddcfd8376fc14a36655af1/pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27", size = 1999404 },
+    { url = "https://files.pythonhosted.org/packages/ca/2d/c365cfa930ed23bc58c41463bae347d1005537dc8db79e998af8ba28d35e/pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee", size = 2130130 },
+    { url = "https://files.pythonhosted.org/packages/f4/d7/eb64d015c350b7cdb371145b54d96c919d4db516817f31cd1c650cae3b21/pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1", size = 2157946 },
+    { url = "https://files.pythonhosted.org/packages/a4/99/bddde3ddde76c03b65dfd5a66ab436c4e58ffc42927d4ff1198ffbf96f5f/pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130", size = 1834387 },
+    { url = "https://files.pythonhosted.org/packages/71/47/82b5e846e01b26ac6f1893d3c5f9f3a2eb6ba79be26eef0b759b4fe72946/pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee", size = 1990453 },
+    { url = "https://files.pythonhosted.org/packages/51/b2/b2b50d5ecf21acf870190ae5d093602d95f66c9c31f9d5de6062eb329ad1/pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b", size = 1885186 },
+    { url = "https://files.pythonhosted.org/packages/46/72/af70981a341500419e67d5cb45abe552a7c74b66326ac8877588488da1ac/pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e", size = 1891159 },
+    { url = "https://files.pythonhosted.org/packages/ad/3d/c5913cccdef93e0a6a95c2d057d2c2cba347815c845cda79ddd3c0f5e17d/pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8", size = 1768331 },
+    { url = "https://files.pythonhosted.org/packages/f6/f0/a3ae8fbee269e4934f14e2e0e00928f9346c5943174f2811193113e58252/pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3", size = 1822467 },
+    { url = "https://files.pythonhosted.org/packages/d7/7a/7bbf241a04e9f9ea24cd5874354a83526d639b02674648af3f350554276c/pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f", size = 1979797 },
+    { url = "https://files.pythonhosted.org/packages/4f/5f/4784c6107731f89e0005a92ecb8a2efeafdb55eb992b8e9d0a2be5199335/pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133", size = 1987839 },
+    { url = "https://files.pythonhosted.org/packages/6d/a7/61246562b651dff00de86a5f01b6e4befb518df314c54dec187a78d81c84/pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc", size = 1998861 },
+    { url = "https://files.pythonhosted.org/packages/86/aa/837821ecf0c022bbb74ca132e117c358321e72e7f9702d1b6a03758545e2/pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50", size = 2116582 },
+    { url = "https://files.pythonhosted.org/packages/81/b0/5e74656e95623cbaa0a6278d16cf15e10a51f6002e3ec126541e95c29ea3/pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9", size = 2151985 },
+    { url = "https://files.pythonhosted.org/packages/63/37/3e32eeb2a451fddaa3898e2163746b0cffbbdbb4740d38372db0490d67f3/pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151", size = 2004715 },
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.12.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 },
+]
+
+[[package]]
+name = "uvloop"
+version = "0.21.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/af/c0/854216d09d33c543f12a44b393c402e89a920b1a0a7dc634c42de91b9cf6/uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3", size = 2492741 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3d/76/44a55515e8c9505aa1420aebacf4dd82552e5e15691654894e90d0bd051a/uvloop-0.21.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f", size = 1442019 },
+    { url = "https://files.pythonhosted.org/packages/35/5a/62d5800358a78cc25c8a6c72ef8b10851bdb8cca22e14d9c74167b7f86da/uvloop-0.21.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d", size = 801898 },
+    { url = "https://files.pythonhosted.org/packages/f3/96/63695e0ebd7da6c741ccd4489b5947394435e198a1382349c17b1146bb97/uvloop-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f38b2e090258d051d68a5b14d1da7203a3c3677321cf32a95a6f4db4dd8b6f26", size = 3827735 },
+    { url = "https://files.pythonhosted.org/packages/61/e0/f0f8ec84979068ffae132c58c79af1de9cceeb664076beea86d941af1a30/uvloop-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87c43e0f13022b998eb9b973b5e97200c8b90823454d4bc06ab33829e09fb9bb", size = 3825126 },
+    { url = "https://files.pythonhosted.org/packages/bf/fe/5e94a977d058a54a19df95f12f7161ab6e323ad49f4dabc28822eb2df7ea/uvloop-0.21.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10d66943def5fcb6e7b37310eb6b5639fd2ccbc38df1177262b0640c3ca68c1f", size = 3705789 },
+    { url = "https://files.pythonhosted.org/packages/26/dd/c7179618e46092a77e036650c1f056041a028a35c4d76945089fcfc38af8/uvloop-0.21.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:67dd654b8ca23aed0a8e99010b4c34aca62f4b7fce88f39d452ed7622c94845c", size = 3800523 },
+    { url = "https://files.pythonhosted.org/packages/57/a7/4cf0334105c1160dd6819f3297f8700fda7fc30ab4f61fbf3e725acbc7cc/uvloop-0.21.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8", size = 1447410 },
+    { url = "https://files.pythonhosted.org/packages/8c/7c/1517b0bbc2dbe784b563d6ab54f2ef88c890fdad77232c98ed490aa07132/uvloop-0.21.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0", size = 805476 },
+    { url = "https://files.pythonhosted.org/packages/ee/ea/0bfae1aceb82a503f358d8d2fa126ca9dbdb2ba9c7866974faec1cb5875c/uvloop-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e", size = 3960855 },
+    { url = "https://files.pythonhosted.org/packages/8a/ca/0864176a649838b838f36d44bf31c451597ab363b60dc9e09c9630619d41/uvloop-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb", size = 3973185 },
+    { url = "https://files.pythonhosted.org/packages/30/bf/08ad29979a936d63787ba47a540de2132169f140d54aa25bc8c3df3e67f4/uvloop-0.21.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6", size = 3820256 },
+    { url = "https://files.pythonhosted.org/packages/da/e2/5cf6ef37e3daf2f06e651aae5ea108ad30df3cb269102678b61ebf1fdf42/uvloop-0.21.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d", size = 3937323 },
+    { url = "https://files.pythonhosted.org/packages/8c/4c/03f93178830dc7ce8b4cdee1d36770d2f5ebb6f3d37d354e061eefc73545/uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c", size = 1471284 },
+    { url = "https://files.pythonhosted.org/packages/43/3e/92c03f4d05e50f09251bd8b2b2b584a2a7f8fe600008bcc4523337abe676/uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2", size = 821349 },
+    { url = "https://files.pythonhosted.org/packages/a6/ef/a02ec5da49909dbbfb1fd205a9a1ac4e88ea92dcae885e7c961847cd51e2/uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d", size = 4580089 },
+    { url = "https://files.pythonhosted.org/packages/06/a7/b4e6a19925c900be9f98bec0a75e6e8f79bb53bdeb891916609ab3958967/uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc", size = 4693770 },
+    { url = "https://files.pythonhosted.org/packages/ce/0c/f07435a18a4b94ce6bd0677d8319cd3de61f3a9eeb1e5f8ab4e8b5edfcb3/uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb", size = 4451321 },
+    { url = "https://files.pythonhosted.org/packages/8f/eb/f7032be105877bcf924709c97b1bf3b90255b4ec251f9340cef912559f28/uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f", size = 4659022 },
+    { url = "https://files.pythonhosted.org/packages/3f/8d/2cbef610ca21539f0f36e2b34da49302029e7c9f09acef0b1c3b5839412b/uvloop-0.21.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281", size = 1468123 },
+    { url = "https://files.pythonhosted.org/packages/93/0d/b0038d5a469f94ed8f2b2fce2434a18396d8fbfb5da85a0a9781ebbdec14/uvloop-0.21.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af", size = 819325 },
+    { url = "https://files.pythonhosted.org/packages/50/94/0a687f39e78c4c1e02e3272c6b2ccdb4e0085fda3b8352fecd0410ccf915/uvloop-0.21.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6", size = 4582806 },
+    { url = "https://files.pythonhosted.org/packages/d2/19/f5b78616566ea68edd42aacaf645adbf71fbd83fc52281fba555dc27e3f1/uvloop-0.21.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816", size = 4701068 },
+    { url = "https://files.pythonhosted.org/packages/47/57/66f061ee118f413cd22a656de622925097170b9380b30091b78ea0c6ea75/uvloop-0.21.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc", size = 4454428 },
+    { url = "https://files.pythonhosted.org/packages/63/9a/0962b05b308494e3202d3f794a6e85abe471fe3cafdbcf95c2e8c713aabd/uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553", size = 4660018 },
+]