Merge remote-tracking branch 'origin/master' into tb_use_gfile

Author: f4hy

.github/CONTRIBUTING.md

@@ -155,7 +155,7 @@ formatting errors. In certain cases, a missing blank line or a wrong indent can
 Run these commands
 
 ```bash
-pip install ".[docs]"
+pip install -r requirements/docs.txt
 cd docs
 make html
 ```

.pyrightconfig.json

@@ -7,7 +7,7 @@
     "pytorch_lightning/__init__.py",
     "pytorch_lightning/callbacks",
     "pytorch_lightning/core",
-    "pytorch_lightning/accelerator_backends",
+    "pytorch_lightning/accelerators",
     "pytorch_lightning/loggers",
     "pytorch_lightning/logging",
     "pytorch_lightning/metrics",

.run_local_tests.sh

@@ -6,11 +6,6 @@ export SLURM_LOCALID=0
 
 # use this to run tests
 rm -rf _ckpt_*
-rm -rf ./tests/save_dir*
-rm -rf ./tests/mlruns_*
-rm -rf ./tests/cometruns*
-rm -rf ./tests/wandb*
-rm -rf ./tests/tests/*
 rm -rf ./lightning_logs
 python -m coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --flake8
 python -m coverage report -m

CHANGELOG.md

@@ -106,6 +106,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Fixed LR finder and `hparams` compatibility ([#2821](https://github.com/PyTorchLightning/pytorch-lightning/pull/2821))
 
+- Fixed `ModelCheckpoint` not saving the latest information when `save_last=True` ([#2881](https://github.com/PyTorchLightning/pytorch-lightning/pull/2881))
+
 ## [0.8.5] - 2020-07-09
 
 ### Added

MANIFEST.in

@@ -26,7 +26,7 @@ exclude tests
 recursive-exclude docs *
 exclude docs
 recursive-include docs/source/_images/logos/ *
-recursive-include docs/source/_images/general/ pl_overview* tf_* tutorial_*
+recursive-include docs/source/_images/general/ pl_overview* tf_* tutorial_* PTL101_*
 
 # Include the Requirements
 recursive-include requirements *.txt

README.md

@@ -88,6 +88,11 @@ Once you do this, you can train on multiple-GPUs, TPUs, CPUs and even in 16-bit
 
 Get started with our [QUICK START PAGE](https://pytorch-lightning.readthedocs.io/en/stable/new-project.html)
 
+---
+### [Tune in for our PyTorch Lightning 101 class with William Falcon and Alfredo Canziani! New episodes every week!](https://www.youtube.com/watch?v=DbESHcCoWbM&list=PLaMu-SDt_RB5NUm67hU2pdE75j6KaIOv2)
+[![IMAGE ALT TEXT HERE](docs/source/_images/general/PTL101_youtube_thumbnail.jpg)](https://www.youtube.com/watch?v=DbESHcCoWbM&list=PLaMu-SDt_RB5NUm67hU2pdE75j6KaIOv2)
+---
+
 ## Refactoring your PyTorch code + benefits + full walk-through
 [![Watch the video](docs/source/_images/general/tutorial_cover.jpg)](https://www.youtube.com/watch?v=QHww1JH7IDU)
 

dockers/cuda-extras/Dockerfile

@@ -39,7 +39,6 @@ RUN apt-get update && \
     && \
 
 # Install AMP
-    # TODO: skip this instrall for PT >= 1.6
     bash install_AMP.sh && \
 # Install all requirements
     pip install -r requirements.txt && \

docs/source/_images/general/PTL101_youtube_thumbnail.jpg

@@ -138,7 +138,7 @@
 exclude_patterns = [
     'api/pytorch_lightning.rst',
     'api/pl_examples.*',
-    'api/pytorch_lightning.accelerator_backends.*',
+    'api/pytorch_lightning.accelerators.*',
     'api/modules.rst',
     'PULL_REQUEST_TEMPLATE.md',
 ]

docs/source/conf.py

@@ -63,6 +63,7 @@ PyTorch Lightning Documentation
    :name: Tutorials
    :caption: Tutorials
 
+   PyTorch Lightning 101 class <https://www.youtube.com/playlist?list=PLaMu-SDt_RB5NUm67hU2pdE75j6KaIOv2>
    From PyTorch to PyTorch Lightning <https://towardsdatascience.com/from-pytorch-to-pytorch-lightning-a-gentle-introduction-b371b7caaf09>
    Video on how to refactor PyTorch into PyTorch Lightning <https://www.youtube.com/watch?v=QHww1JH7IDU>
 

docs/source/index.rst

@@ -4,8 +4,8 @@
 import os
 from argparse import ArgumentParser
 
-from pytorch_lightning import Trainer, seed_everything
 from pl_examples.models.lightning_template import LightningTemplateModel
+from pytorch_lightning import Trainer, seed_everything
 
 seed_everything(234)
 

pl_examples/basic_examples/cpu_template.py

@@ -4,8 +4,8 @@
 import os
 from argparse import ArgumentParser
 
-from pytorch_lightning import Trainer, seed_everything
 from pl_examples.models.lightning_template import LightningTemplateModel
+from pytorch_lightning import Trainer, seed_everything
 
 seed_everything(234)
 

pl_examples/basic_examples/gpu_template.py

@@ -4,8 +4,8 @@
 import os
 from argparse import ArgumentParser
 
-from pytorch_lightning import Trainer, seed_everything
 from pl_examples.models.lightning_template import LightningTemplateModel
+from pytorch_lightning import Trainer, seed_everything
 
 seed_everything(234)
 

pl_examples/basic_examples/multi_node_ddp2_demo.py

@@ -27,13 +27,10 @@
 from tempfile import TemporaryDirectory
 from typing import Optional, Generator, Union
 
-from torch.nn import Module
-
-import pytorch_lightning as pl
 import torch
 import torch.nn.functional as F
-from pytorch_lightning import _logger as log
 from torch import optim
+from torch.nn import Module
 from torch.optim.lr_scheduler import MultiStepLR
 from torch.optim.optimizer import Optimizer
 from torch.utils.data import DataLoader
@@ -42,6 +39,9 @@
 from torchvision.datasets import ImageFolder
 from torchvision.datasets.utils import download_and_extract_archive
 
+import pytorch_lightning as pl
+from pytorch_lightning import _logger as log
+
 BN_TYPES = (torch.nn.BatchNorm1d, torch.nn.BatchNorm2d, torch.nn.BatchNorm3d)
 DATA_URL = 'https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip'
 

pl_examples/domain_templates/computer_vision_fine_tuning.py

@@ -1,9 +1,9 @@
 """
 This example is largely adapted from https://github.com/pytorch/examples/blob/master/imagenet/main.py
 """
-from argparse import ArgumentParser, Namespace
 import os
 import random
+from argparse import ArgumentParser, Namespace
 from collections import OrderedDict
 
 import torch

pl_examples/domain_templates/imagenet.py

@@ -16,12 +16,9 @@
 tensorboard --logdir default
 """
 
-import pytorch_lightning as pl
-
-from typing import Tuple, List
-
 import argparse
 from collections import OrderedDict, deque, namedtuple
+from typing import Tuple, List
 
 import gym
 import numpy as np
@@ -32,6 +29,8 @@
 from torch.utils.data import DataLoader
 from torch.utils.data.dataset import IterableDataset
 
+import pytorch_lightning as pl
+
 
 class DQN(nn.Module):
     """

pl_examples/domain_templates/reinforce_learn_Qnet.py

@@ -1,4 +1,5 @@
 import os
+import random
 from argparse import ArgumentParser, Namespace
 
 import numpy as np
@@ -7,7 +8,6 @@
 import torchvision.transforms as transforms
 from PIL import Image
 from torch.utils.data import DataLoader, Dataset
-import random
 
 import pytorch_lightning as pl
 from pl_examples.models.unet import UNet

pl_examples/domain_templates/semantic_segmentation.py

@@ -12,7 +12,6 @@
 from torch.utils.data import DataLoader
 from torchvision.datasets import MNIST
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.core import LightningModule
 
 

pl_examples/models/lightning_template.py

@@ -1,6 +1,6 @@
 """Root package info."""
 
-__version__ = '0.9.0rc9'
+__version__ = '0.9.0rc11'
 __author__ = 'William Falcon et al.'
 __author_email__ = '[email protected]'
 __license__ = 'Apache-2.0'

pytorch_lightning/__init__.py

@@ -0,0 +1,7 @@
+from pytorch_lightning.accelerators.cpu_backend import CPUBackend
+from pytorch_lightning.accelerators.ddp2_backend import DDP2Backend
+from pytorch_lightning.accelerators.ddp_backend import DDPBackend
+from pytorch_lightning.accelerators.ddp_spawn_backend import DDPSpawnBackend
+from pytorch_lightning.accelerators.dp_backend import DataParallelBackend
+from pytorch_lightning.accelerators.gpu_backend import GPUBackend
+from pytorch_lightning.accelerators.tpu_backend import TPUBackend

pytorch_lightning/accelerator_backends/__init__.py

@@ -22,7 +22,7 @@ def __init__(self, trainer):
 
     def setup(self, model):
         # run through amp wrapper
-        if self.trainer.use_amp:
+        if self.trainer.amp_type:
             raise MisconfigurationException('amp + cpu is not supported.  Please use a GPU option')
 
         # call setup after the ddp process has connected

pytorch_lightning/accelerators/__init__.py

@@ -13,10 +13,12 @@
 # limitations under the License
 
 import os
+
 import torch
-from pytorch_lightning.utilities import NATIVE_AMP_AVALAIBLE
-from pytorch_lightning.utilities.distributed import rank_zero_only
+
 from pytorch_lightning import _logger as log
+from pytorch_lightning.utilities import AMPType
+from pytorch_lightning.utilities.distributed import rank_zero_only
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 
 try:
@@ -30,9 +32,7 @@
 try:
     from apex import amp
 except ImportError:
-    APEX_AVAILABLE = False
-else:
-    APEX_AVAILABLE = True
+    amp = None
 
 
 class DDP2Backend(object):
@@ -133,10 +133,8 @@ def ddp_train(self, process_idx, mp_queue, model, is_master=False, proc_offset=0
         # set model properties before going into wrapper
         self.trainer.copy_trainer_model_properties(model)
 
-        # AMP
-        # run through amp wrapper before going to distributed DP
-        # TODO: remove with dropping NVIDIA AMP support
-        if self.trainer.use_amp and not NATIVE_AMP_AVALAIBLE:
+        # AMP - run through amp wrapper before going to distributed DP
+        if self.trainer.amp_type == AMPType.APEX:
             model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)
             self.trainer.optimizers = optimizers
             self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)

pytorch_lightning/accelerator_backends/cpu_backend.py pytorch_lightning/accelerators/cpu_backend.py

@@ -13,16 +13,18 @@
 # limitations under the License
 
 import os
-import torch
 import subprocess
 import sys
+from os.path import abspath
 from time import sleep
+from typing import Optional
+
 import numpy as np
-from os.path import abspath
-from pytorch_lightning.utilities import NATIVE_AMP_AVALAIBLE
-from pytorch_lightning.utilities.distributed import rank_zero_only
+import torch
+
 from pytorch_lightning import _logger as log
-from typing import Optional
+from pytorch_lightning.utilities import AMPType
+from pytorch_lightning.utilities.distributed import rank_zero_only
 
 try:
     from hydra.utils import to_absolute_path, get_original_cwd
@@ -35,9 +37,7 @@
 try:
     from apex import amp
 except ImportError:
-    APEX_AVAILABLE = False
-else:
-    APEX_AVAILABLE = True
+    amp = None
 
 
 class DDPBackend(object):
@@ -200,10 +200,8 @@ def ddp_train(self, process_idx, mp_queue, model, is_master=False, proc_offset=0
         # set model properties before going into wrapper
         self.trainer.copy_trainer_model_properties(model)
 
-        # AMP
-        # run through amp wrapper before going to distributed DP
-        # TODO: remove with dropping NVIDIA AMP support
-        if self.trainer.use_amp and not NATIVE_AMP_AVALAIBLE:
+        # AMP - run through amp wrapper before going to distributed DP
+        if self.trainer.amp_type == AMPType.APEX:
             model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)
             self.trainer.optimizers = optimizers
             self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)

pytorch_lightning/accelerator_backends/ddp2_backend.py pytorch_lightning/accelerators/ddp2_backend.py

@@ -12,18 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
-import os
 import torch
 import torch.multiprocessing as mp
-from pytorch_lightning.utilities.distributed import rank_zero_only
+
 from pytorch_lightning import _logger as log
+from pytorch_lightning.utilities import AMPType
+from pytorch_lightning.utilities.distributed import rank_zero_only
 
 try:
     from apex import amp
 except ImportError:
-    APEX_AVAILABLE = False
-else:
-    APEX_AVAILABLE = True
+    amp = None
 
 
 class DDPSpawnBackend(object):
@@ -133,11 +132,9 @@ def ddp_train(self, process_idx, mp_queue, model):
         # set model properties before going into wrapper
         self.trainer.copy_trainer_model_properties(model)
 
-        # AMP
+        # AMP -
         # run through amp wrapper before going to distributed DP
-        # TODO: remove with dropping NVIDIA AMP support
-        native_amp_available = hasattr(torch.cuda, "amp") and hasattr(torch.cuda.amp, "autocast")
-        if self.trainer.use_amp and not native_amp_available:
+        if self.trainer.amp_type == AMPType.APEX:
             model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)
             self.trainer.optimizers = optimizers
             self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)

pytorch_lightning/accelerator_backends/ddp_backend.py pytorch_lightning/accelerators/ddp_backend.py

@@ -13,16 +13,16 @@
 # limitations under the License.
 
 import torch
-from pytorch_lightning.utilities.exceptions import MisconfigurationException
-from pytorch_lightning.overrides.data_parallel import LightningDataParallel
 from torch import optim
 
+from pytorch_lightning.overrides.data_parallel import LightningDataParallel
+from pytorch_lightning.utilities import AMPType
+from pytorch_lightning.utilities.exceptions import MisconfigurationException
+
 try:
     from apex import amp
 except ImportError:
-    APEX_AVAILABLE = False
-else:
-    APEX_AVAILABLE = True
+    amp = None
 
 
 class DataParallelBackend(object):
@@ -49,7 +49,7 @@ def setup(self, model):
         self.model_autocast_original_forward = model.forward
 
         # init half precision
-        if self.trainer.use_amp:
+        if self.trainer.amp_type:
             model = self.__init_half_precision(model)
 
         # init torch data parallel
@@ -69,9 +69,7 @@ def __init_torch_data_parallel(self, model):
         return model
 
     def __init_half_precision(self, model):
-        native_amp_available = hasattr(torch.cuda, "amp") and hasattr(torch.cuda.amp, "autocast")
-
-        if native_amp_available:
+        if self.trainer.amp_type == AMPType.NATIVE:
             self.__init_native_amp(model)
         else:
             model = self.__init_nvidia_apex(model)