Run a single wandb instance per TPU run (#4851)

* Run a single wandb instance per TPU run

* wandb: self.is_world_master

* make style
Co-authored-by: Julien Chaumond <chaumond@gmail.com>

src/transformers/trainer.py

@@ -336,7 +336,10 @@ class Trainer:
             WANDB_DISABLED:
                 (Optional): boolean - defaults to false, set to "true" to disable wandb entirely
         """
-        logger.info('Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"')
+        if self.is_world_master():
+            logger.info(
+                'Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"'
+            )
             wandb.init(project=os.getenv("WANDB_PROJECT", "huggingface"), config=vars(self.args))
             # keep track of model topology and gradients
             if os.getenv("WANDB_WATCH") != "false":
@@ -557,6 +560,7 @@ class Trainer:
                 self.tb_writer.add_scalar(k, v, self.global_step)
             self.tb_writer.flush()
         if is_wandb_available():
+            if self.is_world_master():
                 wandb.log(logs, step=self.global_step)
         output = json.dumps({**logs, **{"step": self.global_step}})
         if iterator is not None: