fix for nemo: do not initialize mpu if it is already initialized

megatron/initialize.py

@@ -123,6 +123,9 @@ def _initialize_distributed():
 
     # Set the model-parallel / data-parallel communicators.
     if device_count > 0:
+        if mpu.model_parallel_is_initialized():
+            print('model parallel is already initialized')
+        else:
             mpu.initialize_model_parallel(args.model_parallel_size)