Unverified Commit f6d03f2f authored by Biswa Panda's avatar Biswa Panda Committed by GitHub
Browse files

docs: update examples in document (#897)

parent b627894a
...@@ -34,26 +34,33 @@ The code for the pipeline looks like this: ...@@ -34,26 +34,33 @@ The code for the pipeline looks like this:
```python ```python
# filename: pipeline.py # filename: pipeline.py
from fastapi import FastAPI
from dynamo.sdk import service, dynamo_endpoint, depends from fastapi.responses import StreamingResponse
from pydantic import BaseModel from pydantic import BaseModel
from dynamo.sdk import DYNAMO_IMAGE, depends, dynamo_endpoint, service
class RequestType(BaseModel): class RequestType(BaseModel):
text: str text: str
@service(resources={"cpu": "1"})
class Frontend:
middle = depends(Middle)
@api class ResponseType(BaseModel):
async def generate(self, text: str): text: str
request = RequestType(text=text)
async for response in self.middle.generate(request.model_dump_json()):
yield f"Frontend: {response}" @service(
dynamo={"enabled": True, "namespace": "inference"},
)
class Backend:
@dynamo_endpoint()
async def generate(self, req: RequestType):
text = f"{req.text}-back"
for token in text.split():
yield f"Backend: {token}"
@service( @service(
resources={"cpu": "1"}, dynamo={"enabled": True, "namespace": "inference"},
dynamo={"enabled": True, "namespace": "inference"}
) )
class Middle: class Middle:
backend = depends(Backend) backend = depends(Backend)
...@@ -61,19 +68,29 @@ class Middle: ...@@ -61,19 +68,29 @@ class Middle:
@dynamo_endpoint() @dynamo_endpoint()
async def generate(self, req: RequestType): async def generate(self, req: RequestType):
text = f"{req.text}-mid" text = f"{req.text}-mid"
for token in text.split(): next_request = RequestType(text=text).model_dump_json()
yield f"Mid: {token}" async for response in self.backend.generate(next_request):
yield f"Middle: {response}"
app = FastAPI(title="Hello World!")
@service( @service(
resources={"cpu": "1"}, dynamo={"enabled": True, "namespace": "inference"},
dynamo={"enabled": True, "namespace": "inference"} app=app,
) )
class Backend: class Frontend:
@dynamo_endpoint() middle = depends(Middle)
async def generate(self, req: RequestType):
text = f"{req.text}-back" @dynamo_endpoint(is_api=True)
for token in text.split(): async def generate(self, request: RequestType):
yield f"Backend: {token}" async def content_generator():
async for response in self.middle.generate(request.model_dump_json()):
yield f"Frontend: {response}"
return StreamingResponse(content_generator())
``` ```
You can run this pipeline locally by spinning up ETCD and NATS and then running the pipeline: You can run this pipeline locally by spinning up ETCD and NATS and then running the pipeline:
...@@ -106,3 +123,4 @@ federer-mid-back ...@@ -106,3 +123,4 @@ federer-mid-back
You can find in-depth documentation for the Dynamo SDK [here](./docs/sdk/README.md) and the Dynamo CLI [here](./docs/cli/README.md) You can find in-depth documentation for the Dynamo SDK [here](./docs/sdk/README.md) and the Dynamo CLI [here](./docs/cli/README.md)
Please refer to [hello_world](../../../examples/hello_world/README.md) and [llm](../../../examples/llm/README.md) for examples.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment