Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "cc26cd8139c672016b6a578ea8d02138b53eb193"
Commit 716da5d8 authored by MaximumEntropy's avatar MaximumEntropy
Browse files

Fix mpu init 


Signed-off-by: default avatarMaximumEntropy <sandeep.subramanian.1@umontreal.ca>
parent 0f6bfdeb
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 2 deletions
megatron/initialize.py
View file @ 716da5d8
...@@ -185,8 +185,7 @@ def _initialize_distributed(): ...@@ -185,8 +185,7 @@ def _initialize_distributed():
mpu.initialize_model_parallel(args.tensor_model_parallel_size, mpu.initialize_model_parallel(args.tensor_model_parallel_size,
args.pipeline_model_parallel_size, args.pipeline_model_parallel_size,
args.virtual_pipeline_model_parallel_size, args.virtual_pipeline_model_parallel_size,
args.pipeline_model_parallel_split_rank, args.pipeline_model_parallel_split_rank)
args.untie_embeddings_and_output_weights)
if args.rank == 0: if args.rank == 0:
print(f'> initialized tensor model parallel with size ' print(f'> initialized tensor model parallel with size '
f'{mpu.get_tensor_model_parallel_world_size()}') f'{mpu.get_tensor_model_parallel_world_size()}')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment