From f9b30a9a3e3a439a56a24c887cac36a2a2e5413b Mon Sep 17 00:00:00 2001 From: William Falcon Date: Tue, 23 Jun 2020 20:48:26 -0400 Subject: [PATCH 01/10] fixes hparam logging --- pytorch_lightning/loggers/tensorboard.py | 7 +++++++ tests/loggers/test_tensorboard.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/pytorch_lightning/loggers/tensorboard.py b/pytorch_lightning/loggers/tensorboard.py index 9ecc60292758e..c63a4e6f0fdcf 100644 --- a/pytorch_lightning/loggers/tensorboard.py +++ b/pytorch_lightning/loggers/tensorboard.py @@ -143,6 +143,13 @@ def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> @rank_zero_only def save(self) -> None: super().save() + + try: + self.experiment.flush() + except AttributeError: + # you are using PT version ( Date: Tue, 23 Jun 2020 22:08:42 -0400 Subject: [PATCH 02/10] fixes hparam logging --- pytorch_lightning/loggers/tensorboard.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pytorch_lightning/loggers/tensorboard.py b/pytorch_lightning/loggers/tensorboard.py index c63a4e6f0fdcf..f9b80d19212ef 100644 --- a/pytorch_lightning/loggers/tensorboard.py +++ b/pytorch_lightning/loggers/tensorboard.py @@ -144,12 +144,6 @@ def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> def save(self) -> None: super().save() - try: - self.experiment.flush() - except AttributeError: - # you are using PT version ( Date: Tue, 23 Jun 2020 22:08:59 -0400 Subject: [PATCH 03/10] fixes hparam logging --- pytorch_lightning/loggers/tensorboard.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/loggers/tensorboard.py b/pytorch_lightning/loggers/tensorboard.py index f9b80d19212ef..9ecc60292758e 100644 --- a/pytorch_lightning/loggers/tensorboard.py +++ b/pytorch_lightning/loggers/tensorboard.py @@ -143,7 +143,6 @@ def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> @rank_zero_only def save(self) -> None: super().save() - dir_path = self.log_dir if not os.path.isdir(dir_path): dir_path = self.save_dir From 23ea6db5347142fc4d8322f1b32b32ad5b32d29b Mon Sep 17 00:00:00 2001 From: William Falcon Date: Tue, 23 Jun 2020 22:53:59 -0400 Subject: [PATCH 04/10] fixes hparam logging --- tests/loggers/test_tensorboard.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index 6636c361c0945..62ae7cd998419 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -1,5 +1,6 @@ from argparse import Namespace +import os import pytest import torch @@ -8,6 +9,8 @@ from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.loggers import TensorBoardLogger from tests.base import EvalModelTemplate +from tensorboard.backend.event_processing.event_accumulator import EventAccumulator +import yaml def test_tensorboard_hparams(tmpdir): @@ -16,8 +19,30 @@ def test_tensorboard_hparams(tmpdir): trainer = Trainer(max_epochs=1, default_root_dir=tmpdir) trainer.fit(model) + folder_path = trainer.logger.log_dir + + # make sure yaml is there + with open(os.path.join(folder_path, 'hparams.yaml')) as file: + # The FullLoader parameter handles the conversion from YAML + # scalar values to Python the dictionary format + yaml_params = yaml.load(file, Loader=yaml.FullLoader) + assert yaml_params['b1'] == 0.5 + assert len(yaml_params.keys()) == 10 + + # verify artifacts + assert len(os.listdir(os.path.join(folder_path, 'checkpoints'))) == 1 + + # verify tb logs + event_acc = EventAccumulator(folder_path) + event_acc.Reload() + + hparams_data = b'\x12\x84\x01"\x0b\n\tdrop_prob"\x0c\n\nbatch_size"\r\n\x0bin_features"' \ + b'\x0f\n\rlearning_rate"\x10\n\x0eoptimizer_name"\x0b\n\tdata_root"\x0e\n' \ + b'\x0cout_features"\x0c\n\nhidden_dim"\x04\n\x02b1"\x04\n\x02b2' + + assert event_acc.summary_metadata['_hparams_/experiment'].plugin_data.plugin_name == 'hparams' + assert event_acc.summary_metadata['_hparams_/experiment'].plugin_data.content == hparams_data -# test_tensorboard_hparams('/Users/williamfalcon/Desktop/') def test_tensorboard_automatic_versioning(tmpdir): """Verify that automatic versioning works""" From 5223e8261553fcf96c8ebea76ae6697b7af8a35d Mon Sep 17 00:00:00 2001 From: William Falcon Date: Tue, 23 Jun 2020 23:05:36 -0400 Subject: [PATCH 05/10] fixes hparam logging --- tests/loggers/test_tensorboard.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index 62ae7cd998419..6b6120b780502 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -13,6 +13,7 @@ import yaml +@pytest.importorskip('torch', minversion='1.5.0') def test_tensorboard_hparams(tmpdir): model = EvalModelTemplate() From a3f450697bc6eb2f4a65e554244823f3cbf27d12 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 24 Jun 2020 09:53:46 +0200 Subject: [PATCH 06/10] Apply suggestions from code review --- tests/loggers/test_tensorboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index 6b6120b780502..1a14854174a72 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -13,7 +13,7 @@ import yaml -@pytest.importorskip('torch', minversion='1.5.0') +@pytest.mark.importorskip('torch', minversion='1.5.0') def test_tensorboard_hparams(tmpdir): model = EvalModelTemplate() From 1ec09f6af5456e64920b7e9d93c2d1226174d0c2 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 24 Jun 2020 11:02:17 +0200 Subject: [PATCH 07/10] skipif --- tests/loggers/test_tensorboard.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index 1a14854174a72..5ee97e86f9fef 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -1,19 +1,18 @@ +import os from argparse import Namespace +from packaging import version -import os import pytest import torch +import yaml +from tensorboard.backend.event_processing.event_accumulator import EventAccumulator -import tests.base.utils as tutils from pytorch_lightning import Trainer -from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.loggers import TensorBoardLogger from tests.base import EvalModelTemplate -from tensorboard.backend.event_processing.event_accumulator import EventAccumulator -import yaml -@pytest.mark.importorskip('torch', minversion='1.5.0') +@pytest.mark.skipif(version.parse(torch.__version__) < version.parse('1.5.0'), reason='Minimal PT version is set to 1.5') def test_tensorboard_hparams(tmpdir): model = EvalModelTemplate() From 268a12e1804f4bdc5ee32569160eddaa7eee1a48 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 24 Jun 2020 11:26:04 +0200 Subject: [PATCH 08/10] rename --- tests/loggers/test_tensorboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index 5ee97e86f9fef..2821f62d54e5d 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -13,7 +13,7 @@ @pytest.mark.skipif(version.parse(torch.__version__) < version.parse('1.5.0'), reason='Minimal PT version is set to 1.5') -def test_tensorboard_hparams(tmpdir): +def test_tensorboard_hparams_reload(tmpdir): model = EvalModelTemplate() trainer = Trainer(max_epochs=1, default_root_dir=tmpdir) From 18e6209242853eb6ac77c3ba73930d5c7bdc1617 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 24 Jun 2020 22:30:37 -0400 Subject: [PATCH 09/10] Update test_tensorboard.py --- tests/loggers/test_tensorboard.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index 2821f62d54e5d..d53123771f02b 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -12,7 +12,8 @@ from tests.base import EvalModelTemplate -@pytest.mark.skipif(version.parse(torch.__version__) < version.parse('1.5.0'), reason='Minimal PT version is set to 1.5') +@pytest.mark.skipif(version.parse(torch.__version__) < version.parse('1.5.0'), + reason='Minimal PT version is set to 1.5') def test_tensorboard_hparams_reload(tmpdir): model = EvalModelTemplate() From 3abfe5a3d9451da3196db5dc4912e08ce90fc7eb Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 24 Jun 2020 23:13:18 -0400 Subject: [PATCH 10/10] Update test_tensorboard.py --- tests/loggers/test_tensorboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index d53123771f02b..901fe9339dd21 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -12,7 +12,7 @@ from tests.base import EvalModelTemplate -@pytest.mark.skipif(version.parse(torch.__version__) < version.parse('1.5.0'), +@pytest.mark.skipif(version.parse(torch.__version__) < version.parse('1.5.0'), reason='Minimal PT version is set to 1.5') def test_tensorboard_hparams_reload(tmpdir): model = EvalModelTemplate()