Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OpenDAS
vllm_cscc
Commits
63fe3a70
Unverified
Commit
63fe3a70
authored
Jul 28, 2025
by
Chauncey
Committed by
GitHub
Jul 28, 2025
Browse files
[PD] let p2p nccl toy proxy handle /chat/completions (#21734)
Signed-off-by:
chaunceyjiang
<
chaunceyjiang@gmail.com
>
parent
0ae970ed
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
3 additions
and
2 deletions
+3
-2
examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py
...gated_serving_p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py
+3
-2
No files found.
examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py
View file @
63fe3a70
...
@@ -120,6 +120,7 @@ async def forward_request(url, data, request_id):
...
@@ -120,6 +120,7 @@ async def forward_request(url, data, request_id):
@
app
.
route
(
"/v1/completions"
,
methods
=
[
"POST"
])
@
app
.
route
(
"/v1/completions"
,
methods
=
[
"POST"
])
@
app
.
route
(
"/v1/chat/completions"
,
methods
=
[
"POST"
])
async
def
handle_request
():
async
def
handle_request
():
try
:
try
:
original_request_data
=
await
request
.
get_json
()
original_request_data
=
await
request
.
get_json
()
...
@@ -157,13 +158,13 @@ async def handle_request():
...
@@ -157,13 +158,13 @@ async def handle_request():
# finish prefill
# finish prefill
async
for
_
in
forward_request
(
async
for
_
in
forward_request
(
f
"http://
{
prefill_addr
}
/v1/completions
"
,
prefill_request
,
request_id
f
"http://
{
prefill_addr
}
{
request
.
path
}
"
,
prefill_request
,
request_id
):
):
continue
continue
# return decode
# return decode
generator
=
forward_request
(
generator
=
forward_request
(
f
"http://
{
decode_addr
}
/v1/completions
"
,
original_request_data
,
request_id
f
"http://
{
decode_addr
}
{
request
.
path
}
"
,
original_request_data
,
request_id
)
)
response
=
await
make_response
(
generator
)
response
=
await
make_response
(
generator
)
response
.
timeout
=
None
response
.
timeout
=
None
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment