Allow not specifying --warmup-init-lr

Summary: Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/736

Differential Revision: D16378001

Pulled By: myleott

fbshipit-source-id: 2907f63bcbf7068ceaa48b00096040fa2639e569

fairseq/optim/lr_scheduler/inverse_square_root_schedule.py

@@ -37,7 +37,7 @@ class InverseSquareRootSchedule(FairseqLRScheduler):
             )
         warmup_end_lr = args.lr[0]
         if args.warmup_init_lr < 0:
-            args.warmup_init_lr = warmup_end_lr
+            args.warmup_init_lr = 0 if args.warmup_updates > 0 else warmup_end_lr
 
         # linearly warmup for the first args.warmup_updates
         self.lr_step = (warmup_end_lr - args.warmup_init_lr) / args.warmup_updates