Unverified Commit 3ef74cde authored by emricksini-h's avatar emricksini-h Committed by GitHub
Browse files

[CI][Tracing] Fix race condition by adding server readiness check (#34364)



Attempt to resolve #34284: "Metrics Tracing (2GPU)" fails with a
segmentation fault.
Signed-off-by: default avataremricksini-h <emrick.birivoutin@hcompany.ai>
parent cd81cdb3
...@@ -107,6 +107,22 @@ class FakeTraceService(TraceServiceServicer): ...@@ -107,6 +107,22 @@ class FakeTraceService(TraceServiceServicer):
self.evt.clear() self.evt.clear()
def _wait_for_server_ready(address: str, timeout: float = 5.0) -> bool:
"""Wait for the gRPC server to be ready to accept connections."""
import socket
import time
host, port = address.rsplit(":", 1)
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with socket.create_connection((host, int(port)), timeout=0.5):
return True
except (OSError, ConnectionRefusedError):
time.sleep(0.1)
return False
@pytest.fixture @pytest.fixture
def trace_service() -> Generator[FakeTraceService, None, None]: def trace_service() -> Generator[FakeTraceService, None, None]:
"""Fixture to set up a fake gRPC trace service.""" """Fixture to set up a fake gRPC trace service."""
...@@ -116,6 +132,13 @@ def trace_service() -> Generator[FakeTraceService, None, None]: ...@@ -116,6 +132,13 @@ def trace_service() -> Generator[FakeTraceService, None, None]:
server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS) server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS)
server.start() server.start()
# Wait for the server to be ready to accept connections
if not _wait_for_server_ready(FAKE_TRACE_SERVER_ADDRESS):
server.stop(grace=None)
raise RuntimeError(
f"Fake trace server failed to start on {FAKE_TRACE_SERVER_ADDRESS}"
)
yield service yield service
server.stop(grace=None) server.stop(grace=None)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment