Run on_validation_end only on main process in DDP (#1125)

Co-authored-by: xingzhaolee <[email protected]>

Author: Xing Zhao LEE

pytorch_lightning/callbacks/model_checkpoint.py

@@ -175,6 +175,10 @@ def format_checkpoint_name(self, epoch, metrics, ver=None):
         return filepath
 
     def on_validation_end(self, trainer, pl_module):
+        # only run on main process
+        if trainer.proc_rank != 0:
+            return
+
         metrics = trainer.callback_metrics
         epoch = trainer.current_epoch
         self.epochs_since_last_check += 1