Skip to content

GitLab

Unverified Commit 343437c7 authored by OlivierDehaene's avatar OlivierDehaene Committed by GitHub
Browse files

feat(router): add device and dtype info (#215)

parent ac8c0f6f
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 0 deletions
server/text_generation_server/server.py
View file @ 343437c7
...@@ -26,6 +26,9 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): ...@@ -26,6 +26,9 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
# Force inference mode for the lifetime of TextGenerationService # Force inference mode for the lifetime of TextGenerationService
self._inference_mode_raii_guard = torch._C._InferenceMode(True) self._inference_mode_raii_guard = torch._C._InferenceMode(True)
async def Info(self, request, context):
return self.model.info
async def ServiceDiscovery(self, request, context): async def ServiceDiscovery(self, request, context):
return generate_pb2.ServiceDiscoveryResponse(urls=self.server_urls) return generate_pb2.ServiceDiscoveryResponse(urls=self.server_urls)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment