Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "e05f03ae41540123a99afdacff86d26170c1315d"
Commit 057a1fab authored by Mostofa Patwary's avatar Mostofa Patwary
Browse files

Fixed uninit communication gruop

parent 10c828ed
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
megatron/mpu/mappings.py
View file @ 057a1fab
...@@ -65,7 +65,7 @@ def _gather(input_): ...@@ -65,7 +65,7 @@ def _gather(input_):
tensor_list = [torch.empty_like(input_) for _ in range(world_size)] tensor_list = [torch.empty_like(input_) for _ in range(world_size)]
tensor_list[rank] = input_ tensor_list[rank] = input_
torch.distributed.all_gather(tensor_list, input_, group=group) torch.distributed.all_gather(tensor_list, input_, group=get_model_parallel_group())
# Note: torch.cat already creates a contiguous tensor. # Note: torch.cat already creates a contiguous tensor.
output = torch.cat(tensor_list, dim=last_dim).contiguous() output = torch.cat(tensor_list, dim=last_dim).contiguous()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment