Enabling single-process fallback for examples/imagenet/main_reducer.py

examples/imagenet/main_reducer.py

@@ -307,6 +307,7 @@ def train(train_loader, model, criterion, optimizer, epoch):
         if args.fp16:
             model.zero_grad()
             loss.backward()
+            if args.distributed:
                 reducer.reduce()
             model_grads_to_master_grads(model_params, master_params)
             if args.static_loss_scale != 1:
@@ -317,6 +318,7 @@ def train(train_loader, model, criterion, optimizer, epoch):
         else:
             optimizer.zero_grad()
             loss.backward()
+            if args.distributed:
                 reducer.reduce()
             optimizer.step()