Skip to content

Commit 3ad6169

Browse files
jakubczakonjakubczakon
authored
Neptune Logger Improvements (#1084)
* removed project and experiment from getstate * added tests for closing experiment, updated token in example to user neptuner * updated teoken * Update neptune.py added a link to example experiment * added exmaple experiment link * dropped duplication * flake fixes * merged with master, added changes information to CHANGELOG
1 parent 2232eb3 commit 3ad6169

File tree

3 files changed

+123
-34
lines changed

3 files changed

+123
-34
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
6868

6969
### Changed
7070

71+
- Improved `NeptuneLogger` by adding `close_after_fit` argument to allow logging after training([#908](https://github.com/PyTorchLightning/pytorch-lightning/pull/1084))
7172
- Changed default TQDM to use `tqdm.auto` for prettier outputs in IPython notebooks ([#752](https://github.com/PyTorchLightning/pytorch-lightning/pull/752))
7273
- Changed `pytorch_lightning.logging` to `pytorch_lightning.loggers` ([#767](https://github.com/PyTorchLightning/pytorch-lightning/pull/767))
7374
- Moved the default `tqdm_dict` definition from Trainer to `LightningModule`, so it can be overridden by the user ([#749](https://github.com/PyTorchLightning/pytorch-lightning/pull/749))

‎pytorch_lightning/loggers/neptune.py

+92-33
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""
2-
Log using `neptune-logger <https://www.neptune.ml>`_
2+
Log using `neptune-logger <https://neptune.ai>`_
33
44
.. _neptune:
55
@@ -30,12 +30,13 @@ class NeptuneLogger(LightningLoggerBase):
3030
"""
3131

3232
def __init__(self, api_key: Optional[str] = None, project_name: Optional[str] = None,
33-
offline_mode: bool = False, experiment_name: Optional[str] = None,
33+
close_after_fit: Optional[bool] = True, offline_mode: bool = False,
34+
experiment_name: Optional[str] = None,
3435
upload_source_files: Optional[List[str]] = None, params: Optional[Dict[str, Any]] = None,
3536
properties: Optional[Dict[str, Any]] = None, tags: Optional[List[str]] = None, **kwargs):
3637
r"""
3738
38-
Initialize a neptune.ml logger.
39+
Initialize a neptune.ai logger.
3940
4041
.. note:: Requires either an API Key (online mode) or a local directory path (offline mode)
4142
@@ -44,10 +45,11 @@ def __init__(self, api_key: Optional[str] = None, project_name: Optional[str] =
4445
# ONLINE MODE
4546
from pytorch_lightning.loggers import NeptuneLogger
4647
# arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class
48+
# We are using an api_key for the anonymous user "neptuner" but you can use your own.
4749
4850
neptune_logger = NeptuneLogger(
49-
api_key=os.environ["NEPTUNE_API_TOKEN"],
50-
project_name="USER_NAME/PROJECT_NAME",
51+
api_key="ANONYMOUS"
52+
project_name="shared/pytorch-lightning-integration",
5153
experiment_name="default", # Optional,
5254
params={"max_epochs": 10}, # Optional,
5355
tags=["pytorch-lightning","mlp"] # Optional,
@@ -85,40 +87,91 @@ def any_lightning_module_function_or_hook(...):
8587
self.logger.experiment.log_artifact("model_checkpoint.pt", prediction_image) # log model checkpoint
8688
self.logger.experiment.whatever_neptune_supports(...)
8789
90+
If you want to log objects after the training is finished use close_after_train=False:
91+
92+
.. code-block:: python
93+
94+
neptune_logger = NeptuneLogger(
95+
...
96+
close_after_fit=False,
97+
...)
98+
trainer = Trainer(logger=neptune_logger)
99+
trainer.fit()
100+
101+
# Log test metrics
102+
trainer.test(model)
103+
104+
# Log additional metrics
105+
from sklearn.metrics import accuracy_score
106+
107+
accuracy = accuracy_score(y_true, y_pred)
108+
neptune_logger.experiment.log_metric('test_accuracy', accuracy)
109+
110+
# Log charts
111+
from scikitplot.metrics import plot_confusion_matrix
112+
import matplotlib.pyplot as plt
113+
114+
fig, ax = plt.subplots(figsize=(16, 12))
115+
plot_confusion_matrix(y_true, y_pred, ax=ax)
116+
neptune_logger.experiment.log_image('confusion_matrix', fig)
117+
118+
# Save checkpoints folder
119+
neptune_logger.experiment.log_artifact('my/checkpoints')
120+
121+
# When you are done, stop the experiment
122+
neptune_logger.experiment.stop()
123+
124+
You can go and see an example experiment here:
125+
https://ui.neptune.ai/o/shared/org/pytorch-lightning-integration/e/PYTOR-66/charts
126+
88127
Args:
89-
api_key (str | None): Required in online mode. Neputne API token, found on https://neptune.ml.
128+
api_key: Required in online mode.
129+
Neputne API token, found on https://neptune.ai
90130
Read how to get your API key
91-
https://docs.neptune.ml/python-api/tutorials/get-started.html#copy-api-token.
92-
project_name (str): Required in online mode. Qualified name of a project in a form of
131+
https://docs.neptune.ai/python-api/tutorials/get-started.html#copy-api-token.
132+
It is recommended to keep it in the `NEPTUNE_API_TOKEN`
133+
environment variable and then you can leave `api_key=None`
134+
project_name: Required in online mode. Qualified name of a project in a form of
93135
"namespace/project_name" for example "tom/minst-classification".
94136
If None, the value of NEPTUNE_PROJECT environment variable will be taken.
95-
You need to create the project in https://neptune.ml first.
96-
offline_mode (bool): Optional default False. If offline_mode=True no logs will be send to neptune.
97-
Usually used for debug purposes.
98-
experiment_name (str|None): Optional. Editable name of the experiment.
99-
Name is displayed in the experiment’s Details (Metadata section) and in experiments view as a column.
100-
upload_source_files (list|None): Optional. List of source files to be uploaded.
101-
Must be list of str or single str. Uploaded sources are displayed in the experiment’s Source code tab.
137+
You need to create the project in https://neptune.ai first.
138+
offline_mode: Optional default False. If offline_mode=True no logs will be send
139+
to neptune. Usually used for debug purposes.
140+
close_after_fit: Optional default True. If close_after_fit=False the experiment
141+
will not be closed after training and additional metrics,
142+
images or artifacts can be logged. Also, remember to close the experiment explicitly
143+
by running neptune_logger.experiment.stop().
144+
experiment_name: Optional. Editable name of the experiment.
145+
Name is displayed in the experiment’s Details (Metadata section) and i
146+
n experiments view as a column.
147+
upload_source_files: Optional. List of source files to be uploaded.
148+
Must be list of str or single str. Uploaded sources are displayed
149+
in the experiment’s Source code tab.
102150
If None is passed, Python file from which experiment was created will be uploaded.
103-
Pass empty list ([]) to upload no files. Unix style pathname pattern expansion is supported.
151+
Pass empty list ([]) to upload no files.
152+
Unix style pathname pattern expansion is supported.
104153
For example, you can pass '\*.py'
105154
to upload all python source files from the current directory.
106155
For recursion lookup use '\**/\*.py' (for Python 3.5 and later).
107156
For more information see glob library.
108-
params (dict|None): Optional. Parameters of the experiment. After experiment creation params are read-only.
109-
Parameters are displayed in the experiment’s Parameters section and each key-value pair can be
110-
viewed in experiments view as a column.
111-
properties (dict|None): Optional default is {}. Properties of the experiment.
112-
They are editable after experiment is created. Properties are displayed in the experiment’s Details and
157+
params: Optional. Parameters of the experiment.
158+
After experiment creation params are read-only.
159+
Parameters are displayed in the experiment’s Parameters section and
160+
each key-value pair can be viewed in experiments view as a column.
161+
properties: Optional default is {}. Properties of the experiment.
162+
They are editable after experiment is created.
163+
Properties are displayed in the experiment’s Details and
113164
each key-value pair can be viewed in experiments view as a column.
114-
tags (list|None): Optional default []. Must be list of str. Tags of the experiment.
165+
tags: Optional default []. Must be list of str. Tags of the experiment.
115166
They are editable after experiment is created (see: append_tag() and remove_tag()).
116-
Tags are displayed in the experiment’s Details and can be viewed in experiments view as a column.
167+
Tags are displayed in the experiment’s Details and can be viewed
168+
in experiments view as a column.
117169
"""
118170
super().__init__()
119171
self.api_key = api_key
120172
self.project_name = project_name
121173
self.offline_mode = offline_mode
174+
self.close_after_fit = close_after_fit
122175
self.experiment_name = experiment_name
123176
self.upload_source_files = upload_source_files
124177
self.params = params
@@ -138,6 +191,12 @@ def any_lightning_module_function_or_hook(...):
138191

139192
log.info(f'NeptuneLogger was initialized in {self.mode} mode')
140193

194+
def __getstate__(self):
195+
state = self.__dict__.copy()
196+
# cannot be pickled
197+
state['_experiment'] = None
198+
return state
199+
141200
@property
142201
def experiment(self) -> Experiment:
143202
r"""
@@ -150,15 +209,14 @@ def experiment(self) -> Experiment:
150209
151210
"""
152211

153-
if self._experiment is not None:
154-
return self._experiment
155-
else:
156-
self._experiment = neptune.create_experiment(name=self.experiment_name,
157-
params=self.params,
158-
properties=self.properties,
159-
tags=self.tags,
160-
upload_source_files=self.upload_source_files,
161-
**self._kwargs)
212+
if self._experiment is None:
213+
self._experiment = neptune.create_experiment(
214+
name=self.experiment_name,
215+
params=self.params,
216+
properties=self.properties,
217+
tags=self.tags,
218+
upload_source_files=self.upload_source_files,
219+
**self._kwargs)
162220
return self._experiment
163221

164222
@rank_zero_only
@@ -184,7 +242,8 @@ def log_metrics(
184242

185243
@rank_zero_only
186244
def finalize(self, status: str) -> None:
187-
self.experiment.stop()
245+
if self.close_after_fit:
246+
self.experiment.stop()
188247

189248
@property
190249
def name(self) -> str:

‎tests/loggers/test_neptune.py

+30-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import pickle
2-
from unittest.mock import patch
2+
3+
from unittest.mock import patch, MagicMock
34

45
import torch
56

@@ -96,3 +97,31 @@ def test_neptune_pickle(tmpdir):
9697
pkl_bytes = pickle.dumps(trainer)
9798
trainer2 = pickle.loads(pkl_bytes)
9899
trainer2.logger.log_metrics({'acc': 1.0})
100+
101+
102+
def test_neptune_leave_open_experiment_after_fit(tmpdir):
103+
"""Verify that neptune experiment was closed after training"""
104+
tutils.reset_seed()
105+
106+
hparams = tutils.get_hparams()
107+
model = LightningTestModel(hparams)
108+
109+
def _run_training(logger):
110+
logger._experiment = MagicMock()
111+
112+
trainer_options = dict(
113+
default_save_path=tmpdir,
114+
max_epochs=1,
115+
train_percent_check=0.05,
116+
logger=logger
117+
)
118+
trainer = Trainer(**trainer_options)
119+
trainer.fit(model)
120+
return logger
121+
122+
logger_close_after_fit = _run_training(NeptuneLogger(offline_mode=True))
123+
assert logger_close_after_fit._experiment.stop.call_count == 1
124+
125+
logger_open_after_fit = _run_training(
126+
NeptuneLogger(offline_mode=True, close_after_fit=False))
127+
assert logger_open_after_fit._experiment.stop.call_count == 0

0 commit comments

Comments
 (0)