Skip to content

GitLab

Commit dfd9b5d5 authored by Rick Ho's avatar Rick Ho
Browse files

fix smart schedule shadow sync bug

parent 0a6becae
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 2 deletions
cuda/fastermoe/smart_schedule.h
View file @ dfd9b5d5
...@@ -338,7 +338,7 @@ void fmoe_cuda_fused_backward_impl( ...@@ -338,7 +338,7 @@ void fmoe_cuda_fused_backward_impl(
collect_fn(si, i / num_expert, 0); collect_fn(si, i / num_expert, 0);
if (i / num_expert == rank) { if (i / num_expert == rank) {
cudaEventCreate(evt_reduce + i % num_expert); cudaEventCreate(evt_reduce + i % num_expert);
cudaEventRecord(evt_reduce[i % num_expert], smgr->stream(num_expert)); cudaEventRecord(evt_reduce[i % num_expert], smgr->stream(0));
} }
++si; ++si;
} }
...@@ -367,7 +367,6 @@ void fmoe_cuda_fused_backward_impl( ...@@ -367,7 +367,6 @@ void fmoe_cuda_fused_backward_impl(
for (long i = 0, si = 0; i < world_size * num_expert; ++i) { for (long i = 0, si = 0; i < world_size * num_expert; ++i) {
if (stored_models[i]) { if (stored_models[i]) {
if (i / num_expert == rank) { if (i / num_expert == rank) {
FMOE_SWE(smgr->stream(0), evt_reduce[i % num_expert]);
FMOE_SWE(smgr->torchStream(), evt_reduce[i % num_expert]); FMOE_SWE(smgr->torchStream(), evt_reduce[i % num_expert]);
set_grad_fn(si, i % num_expert); set_grad_fn(si, i % num_expert);
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment