add backup for metrics.csv

Author: AntonioMirarchi

torchmdnet/scripts/train.py

@@ -21,7 +21,7 @@
 from torchmdnet.models import output_modules
 from torchmdnet.models.model import create_prior_models
 from torchmdnet.models.utils import rbf_class_mapping, act_class_mapping, dtype_mapping
-from torchmdnet.utils import LoadFromFile, LoadFromCheckpoint, save_argparse, number
+from torchmdnet.utils import LoadFromFile, LoadFromCheckpoint, save_argparse, number, check_logs
 from lightning_utilities.core.rank_zero import rank_zero_warn
 
 
@@ -178,8 +178,9 @@ def main():
         auto_insert_metric_name=False,
     )
     early_stopping = EarlyStopping(val_loss_name, patience=args.early_stopping_patience)
-
     csv_logger = CSVLogger(args.log_dir, name="", version="")
+    check_logs(csv_logger)
+    
     _logger = [csv_logger]
     if args.wandb_use:
         wandb_logger = WandbLogger(

torchmdnet/utils.py

@@ -397,3 +397,13 @@ def wrapped_init(self, *args, **kwargs):
 
     cls.__init__ = wrapped_init
     return cls
+
+def check_logs(csvlogger):
+    import os 
+    import time
+    metr_file_path = csvlogger.experiment.metrics_file_path
+    if os.path.exists(metr_file_path):
+        # we make a backup of the metrics file (rename)
+        bckp_date = f'{time.strftime("%Y%m%d")}-{time.strftime("%H%M%S")}'
+        os.rename(metr_file_path, metr_file_path.replace(".csv", f"_{bckp_date}.csv"))
+    return