fix ema (#33)

* fix ema

* add assert

---------
Co-authored-by: jixh <jixh@dp.tech>

unicore/trainer.py

@@ -113,13 +113,11 @@ class Trainer(object):
         if args.ema_decay > 0 and (
             self.data_parallel_rank == 0 or args.validate_with_ema
         ):
-            assert isinstance(
-                self.optimizer, optim.FP16Optimizer
-            ), "ema must with fp16 optimizer"
+
+            assert (self.args.fp16 or self.args.bf16), "ema must with fp16 or bf16"
             self.ema = ExponentialMovingAverageModel(
                 model,
                 args.ema_decay,
-                self._optimizer.fp32_params,
             )
 
         else: