Skip to content

Add backup for metrics.csv #345

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jan 28, 2025
Merged

Add backup for metrics.csv #345

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3ab8ccb
add backup for metrics.csv
AntonioMirarchi Oct 16, 2024
c5a05a5
to_black
AntonioMirarchi Oct 16, 2024
4e7cea8
avoid error when multiple gpus are used
AntonioMirarchi Oct 16, 2024
e1bc7ef
Merge branch 'main' into update_csv_logger
AntonioMirarchi Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions torchmdnet/scripts/train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,13 @@
from torchmdnet.models import output_modules
from torchmdnet.models.model import create_prior_models
from torchmdnet.models.utils import rbf_class_mapping, act_class_mapping, dtype_mapping
from torchmdnet.utils import LoadFromFile, LoadFromCheckpoint, save_argparse, number
from torchmdnet.utils import (
LoadFromFile,
LoadFromCheckpoint,
save_argparse,
number,
check_logs,
)
from lightning_utilities.core.rank_zero import rank_zero_warn


Expand Down Expand Up @@ -219,9 +225,11 @@ def main():
args.early_stopping_monitor, patience=args.early_stopping_patience
)
callbacks.append(early_stopping)


check_logs(args.log_dir)
csv_logger = CSVLogger(args.log_dir, name="", version="")
_logger = [csv_logger]

if args.wandb_use:
wandb_logger = WandbLogger(
project=args.wandb_project,
Expand Down
10 changes: 10 additions & 0 deletions torchmdnet/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,3 +397,13 @@ def wrapped_init(self, *args, **kwargs):

cls.__init__ = wrapped_init
return cls

def check_logs(log_dir):
import os
import time
metr_file_path = os.path.join(log_dir, 'metrics.csv')
if os.path.exists(metr_file_path):
# we make a backup of the metrics file (rename)
bckp_date = f'{time.strftime("%Y%m%d")}-{time.strftime("%H%M%S")}'
os.rename(metr_file_path, metr_file_path.replace(".csv", f"_{bckp_date}.csv"))
return
Loading